Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(syncer): Add REST API connector (#18256)

mkaaadc3b977f7b6a8
diff --git a/internal/handler/connector.go b/internal/handler/connector.go
index 796f2ae9c0..a8ab41c8e5 100644
--- a/internal/handler/connector.go
+++ b/internal/handler/connector.go
@@ -29,6 +29,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/entity"
"ragflow/internal/service"
+ syncerconnector "ragflow/internal/syncer/connector"
)
type connectorServiceIface interface {
@@ -361,8 +362,17 @@ func (h *ConnectorHandler) TestConnector(c *gin.Context) {
return
}
if err != nil && !errors.Is(err, service.ErrConnectorNoAuth) && !errors.Is(err, service.ErrConnectorNotFound) {
- // Validation failure (e.g. missing credentials): mirror Python's DATA_ERROR with data=false.
- common.ResponseWithCodeData(c, common.CodeDataError, false, err.Error())
+ // Schema/credential validation failures map to DATA_ERROR;
+ // anything unexpected falls back to SERVER_ERROR.
+ var (
+ valErr *syncerconnector.ConnectorValidationError
+ credErr *syncerconnector.ConnectorMissingCredentialError
+ )
+ if errors.As(err, &valErr) || errors.As(err, &credErr) {
+ common.ResponseWithCodeData(c, common.CodeDataError, false, err.Error())
+ return
+ }
+ common.ResponseWithCodeData(c, common.CodeServerError, false, "REST API connector validation failed, please check logs.")
return
}
if connectorErrorResponse(c, err) {
diff --git a/internal/handler/connector_test.go b/internal/handler/connector_test.go
index 3d420bf5a1..3b2d744a44 100644
--- a/internal/handler/connector_test.go
+++ b/internal/handler/connector_test.go
@@ -15,6 +15,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/entity"
"ragflow/internal/service"
+ syncerconnector "ragflow/internal/syncer/connector"
)
type fakeConnectorService struct {
@@ -265,10 +266,20 @@ func TestConnectorHandlerTestConnector(t *testing.T) {
wantCode: common.CodeArgumentError,
},
{
- name: "validation failure",
- err: fmt.Errorf("connector credentials are missing"),
+ name: "schema validation failure",
+ err: &syncerconnector.ConnectorValidationError{Message: "At least one content field must be configured (content_fields)."},
wantCode: common.CodeDataError,
},
+ {
+ name: "missing credential failure",
+ err: &syncerconnector.ConnectorMissingCredentialError{Message: "REST API (bearer) requires 'token' in credentials"},
+ wantCode: common.CodeDataError,
+ },
+ {
+ name: "unexpected failure",
+ err: fmt.Errorf("boom"),
+ wantCode: common.CodeServerError,
+ },
}
for _, tt := range tests {
diff --git a/internal/syncer/connector/rest_api.go b/internal/syncer/connector/rest_api.go
new file mode 100644
index 0000000000..9d08756340
--- /dev/null
+++ b/internal/syncer/connector/rest_api.go
@@ -0,0 +1,2211 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "crypto/tls"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "math/rand"
+ "net"
+ "net/http"
+ "net/url"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+ "unicode"
+)
+
+// RestAPIConnector is a configuration-driven REST API data source connector.
+//
+// It mirrors common/data_source/rest_api_connector.py: the same config schema,
+// auth derivation, SSRF protection, pagination, JSON extraction and
+// item-to-document mapping. The connector is read-only over HTTP(S).
+type RestAPIConnector struct {
+ cfg *restAPIConfig
+ baseURL string
+ urlParams map[string]string
+ credentials map[string]any
+
+ authHeaders map[string]string
+ basicAuth *restAPIBasicAuth
+ prepared bool
+}
+
+// restAPIConfig is the validated connector configuration.
+type restAPIConfig struct {
+ URL string
+ Method string
+ Headers map[string]string
+ QueryParams map[string]string
+ AuthType string
+ AuthConfig map[string]any
+ ItemsPath string
+ IDField string
+ ContentFields []string
+ MetadataFields []string
+ PaginationType string
+ PaginationConfig map[string]any
+ PollTimestampField string
+ RequestBody map[string]any
+ FieldTypeHints map[string]string
+ FieldDefaultValues map[string]any
+ ContentTemplate string
+ BatchSize int
+ MaxPages int
+ RequestDelay float64
+}
+
+// restAPIBasicAuth carries HTTP basic credentials.
+type restAPIBasicAuth struct {
+ username string
+ password string
+}
+
+const (
+ restAPIDefaultBatchSize = 2 // default per-batch document count
+ restAPIDefaultMaxPages = 1000 // default page fetch cap
+ restAPIDefaultRequestDelay = 0.5
+ restAPIRequestTimeout = 60 * time.Second
+ restAPIMaxRedirects = 5
+)
+
+const (
+ restAPIAuthNone = "none"
+ restAPIAuthAPIKeyHeader = "api_key_header"
+ restAPIAuthBearer = "bearer"
+ restAPIAuthBasic = "basic"
+
+ restAPIPaginationNone = "none"
+ restAPIPaginationPage = "page"
+ restAPIPaginationOffset = "offset"
+ restAPIPaginationCursor = "cursor"
+)
+
+// Connector error types let callers (e.g. the connector /test endpoint) map
+// failures to stable response codes.
+type (
+ // ConnectorValidationError mirrors ConnectorValidationError.
+ ConnectorValidationError struct{ Message string }
+ // ConnectorMissingCredentialError mirrors ConnectorMissingCredentialError.
+ ConnectorMissingCredentialError struct{ Message string }
+ // RateLimitTriedTooManyTimesError mirrors RateLimitTriedTooManyTimesError.
+ RateLimitTriedTooManyTimesError struct{ Message string }
+)
+
+func (e *ConnectorValidationError) Error() string { return e.Message }
+func (e *ConnectorMissingCredentialError) Error() string { return e.Message }
+func (e *RateLimitTriedTooManyTimesError) Error() string { return e.Message }
+
+// ErrPruneUnsupported is returned by OpenPrune for connectors that do not
+// support deleted-file pruning; prune tasks then complete without deleting
+// anything.
+var ErrPruneUnsupported = errors.New("rest API connector does not support prune")
+
+// Retry/backoff knobs for page fetches and rate limiting. They are package
+// variables so tests can shrink the delays.
+var (
+ restAPIRetryTries = 5
+ restAPIRetryBaseDelay = time.Second
+ restAPIRetryMaxDelay = 30 * time.Second
+ restAPIRetryBackoff = 2
+ restAPIRetryJitter = time.Second
+ restAPI429MaxWaits = 30
+ restAPI429DefaultWait = 30 * time.Second
+)
+
+// restAPISSRFAllowLoopback is a test hook that lets unit tests exercise the
+// real HTTP path against httptest servers bound to loopback. Production code
+// keeps it false so loopback/private endpoints stay blocked.
+var restAPISSRFAllowLoopback bool
+
+// NewRestAPIConnector parses a connector config and returns a connector. It
+// performs schema validation and the base-URL SSRF check but performs no
+// network I/O. Credentials are read from config["credentials"].
+func NewRestAPIConnector(config map[string]any) (*RestAPIConnector, error) {
+ cfg, err := parseRestAPIConfig(config)
+ if err != nil {
+ return nil, err
+ }
+ if err := validateRestAPIURLForSSRF(cfg.URL); err != nil {
+ return nil, err
+ }
+
+ parsed, err := url.Parse(cfg.URL)
+ if err != nil {
+ return nil, &ConnectorValidationError{Message: "Invalid REST API config: url"}
+ }
+
+ // Keep the raw path so "{key}" template placeholders survive URL parsing
+ // (url.URL.String() would percent-encode them).
+ path := parsed.RawPath
+ if path == "" {
+ path = parsed.Path
+ }
+ c := &RestAPIConnector{
+ cfg: cfg,
+ baseURL: parsed.Scheme + "://" + parsed.Host + path,
+ urlParams: restAPIURLQueryParams(parsed),
+ credentials: restAPICredentialsFromConfig(config),
+ authHeaders: map[string]string{},
+ }
+ return c, nil
+}
+
+// restAPICredentialsFromConfig extracts and normalizes stored credentials.
+func restAPICredentialsFromConfig(config map[string]any) map[string]any {
+ raw, ok := config["credentials"]
+ if !ok || raw == nil {
+ return map[string]any{}
+ }
+ if creds, ok := raw.(map[string]any); ok {
+ return creds
+ }
+ if text, ok := raw.(string); ok {
+ var creds map[string]any
+ if json.Unmarshal([]byte(text), &creds) == nil && creds != nil {
+ return creds
+ }
+ }
+ return map[string]any{}
+}
+
+// parseRestAPIConfig mirrors RestAPIConnectorConfig + parse_storage_config.
+func parseRestAPIConfig(config map[string]any) (*restAPIConfig, error) {
+ cfg := &restAPIConfig{
+ Method: "GET",
+ AuthType: restAPIAuthNone,
+ PaginationType: restAPIPaginationNone,
+ BatchSize: restAPIDefaultBatchSize,
+ MaxPages: restAPIDefaultMaxPages,
+ RequestDelay: restAPIDefaultRequestDelay,
+ }
+
+ cfg.URL = strings.TrimSpace(stringConfig(config["url"]))
+ if cfg.URL == "" {
+ return nil, &ConnectorValidationError{Message: "Invalid REST API config: url\n Field required"}
+ }
+ if method := strings.ToUpper(strings.TrimSpace(stringConfig(config["method"]))); method != "" {
+ cfg.Method = method
+ }
+ if cfg.Method != "GET" && cfg.Method != "POST" {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported HTTP method '%s'.", cfg.Method)}
+ }
+ cfg.Headers = restAPITextToDict(config["headers"])
+ cfg.QueryParams = restAPITextToDict(config["query_params"])
+
+ if auth := strings.TrimSpace(stringConfig(config["auth_type"])); auth != "" {
+ cfg.AuthType = auth
+ }
+ switch cfg.AuthType {
+ case restAPIAuthNone, restAPIAuthAPIKeyHeader, restAPIAuthBearer, restAPIAuthBasic:
+ default:
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported auth_type '%s'.", cfg.AuthType)}
+ }
+ cfg.AuthConfig = configAnyMap(config["auth_config"])
+ cfg.ItemsPath = strings.TrimSpace(stringConfig(config["items_path"]))
+ cfg.IDField = strings.TrimSpace(stringConfig(config["id_field"]))
+ cfg.ContentFields = restAPICoerceFieldList(config["content_fields"])
+ if len(cfg.ContentFields) == 0 {
+ return nil, &ConnectorValidationError{Message: "At least one content field must be configured (content_fields)."}
+ }
+ cfg.MetadataFields = restAPICoerceFieldList(config["metadata_fields"])
+
+ if pagination := strings.TrimSpace(stringConfig(config["pagination_type"])); pagination != "" {
+ cfg.PaginationType = pagination
+ }
+ switch cfg.PaginationType {
+ case restAPIPaginationNone, restAPIPaginationPage, restAPIPaginationOffset, restAPIPaginationCursor:
+ default:
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported pagination_type '%s'.", cfg.PaginationType)}
+ }
+ cfg.PaginationConfig = configAnyMap(config["pagination_config"])
+ cfg.PollTimestampField = strings.TrimSpace(stringConfig(config["poll_timestamp_field"]))
+
+ cfg.RequestBody = configAnyMap(config["request_body"])
+ if len(cfg.RequestBody) == 0 {
+ cfg.RequestBody = configAnyMap(cfg.PaginationConfig["request_body"])
+ }
+ cfg.FieldTypeHints = configStringMap(config["field_type_hints"])
+ cfg.FieldDefaultValues = configAnyMap(config["field_default_values"])
+ cfg.ContentTemplate = stringConfig(config["content_template"])
+ if v, ok := restAPIConfigInt(config["batch_size"]); ok {
+ cfg.BatchSize = v
+ }
+ if cfg.BatchSize <= 0 {
+ cfg.BatchSize = restAPIDefaultBatchSize
+ }
+ if v, ok := restAPIConfigInt(config["max_pages"]); ok {
+ if v <= 0 {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Invalid REST API config: max_pages must be a positive integer, got %d", v)}
+ }
+ cfg.MaxPages = v
+ }
+ if v := restAPIConfigFloat(config["request_delay"]); v >= 0 {
+ cfg.RequestDelay = v
+ }
+ return cfg, nil
+}
+
+// restAPITextToDict mirrors _text_to_dict: dict, JSON string, or key=value
+// lines are all accepted.
+func restAPITextToDict(value any) map[string]string {
+ if value == nil {
+ return nil
+ }
+ if dict, ok := value.(map[string]any); ok {
+ out := make(map[string]string, len(dict))
+ for k, v := range dict {
+ out[k] = fmt.Sprint(v)
+ }
+ return out
+ }
+ text, ok := value.(string)
+ if !ok || strings.TrimSpace(text) == "" {
+ return nil
+ }
+ var parsed map[string]any
+ if json.Unmarshal([]byte(text), &parsed) == nil && parsed != nil {
+ out := make(map[string]string, len(parsed))
+ for k, v := range parsed {
+ out[k] = fmt.Sprint(v)
+ }
+ return out
+ }
+ out := map[string]string{}
+ for _, line := range strings.Split(text, "\n") {
+ line = strings.TrimSpace(line)
+ if line == "" || strings.HasPrefix(line, "#") {
+ continue
+ }
+ if idx := strings.Index(line, "="); idx >= 0 {
+ out[strings.TrimSpace(line[:idx])] = strings.TrimSpace(line[idx+1:])
+ }
+ }
+ return out
+}
+
+// restAPICoerceFieldList mirrors the content_fields/metadata_fields coercion.
+func restAPICoerceFieldList(value any) []string {
+ if value == nil {
+ return nil
+ }
+ if text, ok := value.(string); ok {
+ text = strings.TrimSpace(text)
+ if text == "" {
+ return nil
+ }
+ parts := strings.Split(text, ",")
+ out := make([]string, 0, len(parts))
+ for _, p := range parts {
+ if p = strings.TrimSpace(p); p != "" {
+ out = append(out, p)
+ }
+ }
+ return out
+ }
+ if list, ok := value.([]any); ok {
+ out := make([]string, 0, len(list))
+ for _, item := range list {
+ if s := strings.TrimSpace(fmt.Sprint(item)); s != "" {
+ out = append(out, s)
+ }
+ }
+ return out
+ }
+ return nil
+}
+
+func configAnyMap(value any) map[string]any {
+ if m, ok := value.(map[string]any); ok {
+ return m
+ }
+ return nil
+}
+
+func configStringMap(value any) map[string]string {
+ if m, ok := value.(map[string]any); ok {
+ out := make(map[string]string, len(m))
+ for k, v := range m {
+ out[k] = stringConfig(v)
+ }
+ return out
+ }
+ return nil
+}
+
+// restAPIConfigInt reads an integer config value without the positive-only
+// coercion of configInt so values like max_pages=0 round-trip faithfully.
+func restAPIConfigInt(value any) (int, bool) {
+ switch typed := value.(type) {
+ case int:
+ return typed, true
+ case int64:
+ return int(typed), true
+ case float64:
+ return int(typed), true
+ case json.Number:
+ if v, err := typed.Int64(); err == nil {
+ return int(v), true
+ }
+ case string:
+ if v, err := strconv.Atoi(strings.TrimSpace(typed)); err == nil {
+ return v, true
+ }
+ }
+ return 0, false
+}
+
+func restAPIConfigFloat(value any) float64 {
+ switch typed := value.(type) {
+ case float64:
+ return typed
+ case int:
+ return float64(typed)
+ case int64:
+ return float64(typed)
+ case json.Number:
+ if v, err := typed.Float64(); err == nil {
+ return v
+ }
+ case string:
+ if v, err := strconv.ParseFloat(strings.TrimSpace(typed), 64); err == nil {
+ return v
+ }
+ }
+ return -1
+}
+
+// restAPIURLQueryParams splits the URL query string into a string map, keeping
+// the last value per key.
+func restAPIURLQueryParams(parsed *url.URL) map[string]string {
+ if parsed.RawQuery == "" {
+ return nil
+ }
+ values := parsed.Query()
+ out := make(map[string]string, len(values))
+ for k, list := range values {
+ if len(list) > 0 {
+ out[k] = list[len(list)-1]
+ }
+ }
+ return out
+}
+
+// prepare builds auth headers from credentials, mirroring _build_auth.
+func (c *RestAPIConnector) prepare() error {
+ if c.prepared {
+ return nil
+ }
+ c.authHeaders = map[string]string{}
+ c.basicAuth = nil
+
+ switch c.cfg.AuthType {
+ case restAPIAuthNone:
+ case restAPIAuthAPIKeyHeader:
+ headerName := strings.TrimSpace(stringConfig(c.cfg.AuthConfig["header_name"]))
+ apiKey := stringConfig(c.credentials["api_key"])
+ if apiKey == "" {
+ apiKey = stringConfig(c.cfg.AuthConfig["api_key_value"])
+ }
+ if apiKey == "" {
+ apiKey = stringConfig(c.cfg.AuthConfig["api_key"])
+ }
+ if headerName == "" || apiKey == "" {
+ return &ConnectorMissingCredentialError{Message: "REST API (api_key_header) requires 'header_name' in auth_config and 'api_key' in credentials"}
+ }
+ c.authHeaders[headerName] = apiKey
+ case restAPIAuthBearer:
+ token := stringConfig(c.credentials["token"])
+ if token == "" {
+ token = stringConfig(c.cfg.AuthConfig["token"])
+ }
+ if token == "" {
+ return &ConnectorMissingCredentialError{Message: "REST API (bearer) requires 'token' in credentials"}
+ }
+ c.authHeaders["Authorization"] = "Bearer " + token
+ case restAPIAuthBasic:
+ username := stringConfig(c.credentials["username"])
+ if username == "" {
+ username = stringConfig(c.cfg.AuthConfig["username"])
+ }
+ password := stringConfig(c.credentials["password"])
+ if password == "" {
+ password = stringConfig(c.cfg.AuthConfig["password"])
+ }
+ if username == "" || password == "" {
+ return &ConnectorMissingCredentialError{Message: "REST API (basic) requires 'username' and 'password'"}
+ }
+ c.basicAuth = &restAPIBasicAuth{username: username, password: password}
+ default:
+ return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported auth_type: %s", c.cfg.AuthType)}
+ }
+ c.prepared = true
+ return nil
+}
+
+// Validate validates configuration and credential presence without network
+// I/O; live connectivity validation is performed by ValidateLive.
+func (c *RestAPIConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return &ConnectorValidationError{Message: "rest API connector is nil"}
+ }
+ return c.prepare()
+}
+
+// ValidateLive mirrors RestAPIConnector.validate_config: schema validation plus
+// a live single-page fetch with the stored credentials.
+func (c *RestAPIConnector) ValidateLive(ctx context.Context) error {
+ if c == nil {
+ return &ConnectorValidationError{Message: "rest API connector is nil"}
+ }
+ if err := c.prepare(); err != nil {
+ return err
+ }
+ params := map[string]any{}
+ switch c.cfg.PaginationType {
+ case restAPIPaginationPage:
+ page := 1
+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["start_page"]); ok {
+ page = v
+ }
+ perPage, err := c.resolvePageSize()
+ if err != nil {
+ return err
+ }
+ c.applyPagePagination(params, page, perPage)
+ case restAPIPaginationOffset:
+ perPage, err := c.resolvePageSize()
+ if err != nil {
+ return err
+ }
+ offset := 0
+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["start_offset"]); ok {
+ offset = v
+ }
+ limit := perPage
+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["limit"]); ok {
+ limit = v
+ }
+ if limit <= 0 {
+ limit = perPage
+ }
+ c.applyOffsetPagination(params, offset, limit)
+ case restAPIPaginationCursor:
+ if cursor := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["initial_cursor"])); cursor != "" {
+ c.applyCursorPagination(params, cursor)
+ }
+ }
+ _, err := c.fetchPage(ctx, params)
+ return err
+}
+
+// OpenSync opens one sync session. Incremental runs filter documents to the
+// request window (start <= updated_at < end).
+func (c *RestAPIConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ if err := c.prepare(); err != nil {
+ return nil, err
+ }
+ var windowStart *time.Time
+ if !request.FromBeginning && request.WindowStart != nil {
+ ws := request.WindowStart.UTC()
+ windowStart = &ws
+ }
+ session := &restAPISyncSession{
+ connector: c,
+ iterator: newRestAPIItemIterator(c),
+ batchSize: c.cfg.BatchSize,
+ fromBeginning: request.FromBeginning,
+ windowStart: windowStart,
+ windowEnd: request.WindowEnd.UTC(),
+ }
+ session.applyResume(request.Resume)
+ return session, nil
+}
+
+// OpenPrune reports prune as unsupported for sources that cannot enumerate a
+// complete slim snapshot; prune tasks then complete as no-ops.
+func (c *RestAPIConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ return nil, ErrPruneUnsupported
+}
+
+// ---------------------------------------------------------------------------
+// HTTP fetching
+// ---------------------------------------------------------------------------
+
+// fetchPage fetches a single page with retry, mirroring _fetch_page.
+func (c *RestAPIConnector) fetchPage(ctx context.Context, params map[string]any) (any, error) {
+ delay := restAPIRetryBaseDelay
+ var lastErr error
+ for attempt := 0; attempt < restAPIRetryTries; attempt++ {
+ response, err := c.fetchPageOnce(ctx, params)
+ if err == nil {
+ return response, nil
+ }
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ var (
+ valErr *ConnectorValidationError
+ credErr *ConnectorMissingCredentialError
+ rateErr *RateLimitTriedTooManyTimesError
+ )
+ if errors.As(err, &valErr) || errors.As(err, &credErr) || errors.As(err, &rateErr) {
+ return nil, err
+ }
+ lastErr = err
+ if attempt == restAPIRetryTries-1 {
+ break
+ }
+ sleepFor := delay + time.Duration(rand.Float64()*float64(restAPIRetryJitter))
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-time.After(sleepFor):
+ }
+ delay = time.Duration(float64(delay) * float64(restAPIRetryBackoff))
+ if delay > restAPIRetryMaxDelay {
+ delay = restAPIRetryMaxDelay
+ }
+ }
+ return nil, lastErr
+}
+
+// fetchPageOnce performs one page fetch including the 429 wait loop and manual
+// redirect handling with per-hop SSRF validation and DNS pinning.
+func (c *RestAPIConnector) fetchPageOnce(ctx context.Context, params map[string]any) (any, error) {
+ if err := c.prepare(); err != nil {
+ return nil, err
+ }
+ merged := map[string]any{}
+ for k, v := range c.urlParams {
+ merged[k] = v
+ }
+ for k, v := range c.cfg.QueryParams {
+ merged[k] = v
+ }
+ for k, v := range params {
+ merged[k] = v
+ }
+
+ currentURL, queryParams := c.buildURLWithTemplates(merged)
+ currentMethod := c.cfg.Method
+ currentHeaders := make(map[string]string, len(c.cfg.Headers)+len(c.authHeaders))
+ for k, v := range c.cfg.Headers {
+ currentHeaders[k] = v
+ }
+ for k, v := range c.authHeaders {
+ currentHeaders[k] = v
+ }
+ currentBody := c.cfg.RequestBody
+ currentAuth := c.basicAuth
+ previousNetloc := restAPINetloc(currentURL)
+
+ for hop := 0; hop <= restAPIMaxRedirects; hop++ {
+ hostname, pinIP, err := assertRestAPIURLSafe(ctx, currentURL)
+ if err != nil {
+ return nil, &ConnectorValidationError{Message: "Unsafe REST API URL: " + err.Error()}
+ }
+
+ var resp *http.Response
+ var respErr error
+ for wait := 0; wait < restAPI429MaxWaits; wait++ {
+ resp, respErr = c.doRequest(ctx, currentMethod, currentURL, currentHeaders, queryParams, currentBody, currentAuth, hostname, pinIP)
+ if respErr != nil {
+ return nil, respErr
+ }
+ if resp.StatusCode != http.StatusTooManyRequests {
+ break
+ }
+ retryAfter := restAPI429DefaultWait
+ if raw := resp.Header.Get("Retry-After"); raw != "" {
+ if seconds, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && seconds >= 0 {
+ retryAfter = time.Duration(seconds) * time.Second
+ }
+ }
+ resp.Body.Close()
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-time.After(retryAfter):
+ }
+ }
+ if respErr != nil {
+ return nil, respErr
+ }
+ if resp.StatusCode == http.StatusTooManyRequests {
+ resp.Body.Close()
+ return nil, &RateLimitTriedTooManyTimesError{Message: fmt.Sprintf("REST API rate limited: exceeded '%d' retries (too many requests)", restAPI429MaxWaits)}
+ }
+
+ if restAPIIsRedirect(resp.StatusCode) {
+ location := resp.Header.Get("Location")
+ if location == "" {
+ return c.handleResponse(resp)
+ }
+ nextURL, err := restAPIResolveURL(currentURL, location)
+ if err != nil {
+ resp.Body.Close()
+ return nil, &ConnectorValidationError{Message: "Unsafe REST API URL: " + err.Error()}
+ }
+ nextNetloc := restAPINetloc(nextURL)
+ if nextNetloc != "" && nextNetloc != previousNetloc {
+ currentHeaders = restAPIStripAuthHeaders(currentHeaders)
+ currentAuth = nil
+ }
+ previousNetloc = nextNetloc
+ if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusSeeOther {
+ currentMethod = "GET"
+ currentBody = nil
+ queryParams = nil
+ }
+ resp.Body.Close()
+ currentURL = nextURL
+ continue
+ }
+ return c.handleResponse(resp)
+ }
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Exceeded %d redirects fetching %q", restAPIMaxRedirects, currentURL)}
+}
+
+func (c *RestAPIConnector) doRequest(
+ ctx context.Context,
+ method, rawURL string,
+ headers map[string]string,
+ params map[string]any,
+ body map[string]any,
+ auth *restAPIBasicAuth,
+ hostname string,
+ pinIP net.IP,
+) (*http.Response, error) {
+ transport := newRestAPIPinnedTransport(hostname, pinIP)
+ client := &http.Client{
+ Transport: transport,
+ Timeout: restAPIRequestTimeout,
+ // Redirects are handled manually so auth headers can be stripped on
+ // cross-origin hops and each hop is re-validated against SSRF.
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ return http.ErrUseLastResponse
+ },
+ }
+
+ fullURL := rawURL
+ if encoded := restAPIEncodeParams(params); encoded != "" {
+ sep := "?"
+ if strings.Contains(rawURL, "?") {
+ sep = "&"
+ }
+ fullURL = rawURL + sep + encoded
+ }
+
+ var req *http.Request
+ var err error
+ switch method {
+ case "GET":
+ req, err = http.NewRequestWithContext(ctx, http.MethodGet, fullURL, nil)
+ case "POST":
+ var payload []byte
+ if body == nil {
+ payload = []byte("{}")
+ } else {
+ payload, err = json.Marshal(body)
+ if err != nil {
+ return nil, &ConnectorValidationError{Message: "REST API request body is not valid JSON"}
+ }
+ }
+ req, err = http.NewRequestWithContext(ctx, http.MethodPost, fullURL, strings.NewReader(string(payload)))
+ if err == nil {
+ req.Header.Set("Content-Type", "application/json")
+ }
+ default:
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported HTTP method: %s", method)}
+ }
+ if err != nil {
+ return nil, err
+ }
+ for k, v := range headers {
+ req.Header.Set(k, v)
+ }
+ if auth != nil {
+ req.SetBasicAuth(auth.username, auth.password)
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ transport.CloseIdleConnections()
+ return nil, err
+ }
+ resp.Body = &restAPICloseIdleBody{body: resp.Body, transport: transport}
+ return resp, nil
+}
+
+// restAPICloseIdleBody closes the underlying response body and then releases
+// the pinned transport's idle connections, so the per-request transports do
+// not leak keep-alive sockets during long paginated syncs.
+type restAPICloseIdleBody struct {
+ body io.ReadCloser
+ transport *http.Transport
+}
+
+func (b *restAPICloseIdleBody) Read(p []byte) (int, error) { return b.body.Read(p) }
+
+func (b *restAPICloseIdleBody) Close() error {
+ err := b.body.Close()
+ b.transport.CloseIdleConnections()
+ return err
+}
+
+func (c *RestAPIConnector) handleResponse(resp *http.Response) (any, error) {
+ defer resp.Body.Close()
+ status := resp.StatusCode
+ switch {
+ case status == http.StatusUnauthorized || status == http.StatusForbidden:
+ return nil, &ConnectorMissingCredentialError{Message: fmt.Sprintf("REST API authentication failed with status %d", status)}
+ case status >= 400 && status < 500 && status != http.StatusTooManyRequests:
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("REST API request failed with non-retriable client error status %d", status)}
+ case status >= 500:
+ // The "http <status>" wording lets the syncer's task-level retry
+ // classifier (isTransientSyncError) recognize exhausted server errors.
+ return nil, fmt.Errorf("REST API request failed with http %d", status)
+ }
+
+ raw, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ decoder := json.NewDecoder(strings.NewReader(string(raw)))
+ decoder.UseNumber()
+ var value any
+ if err := decoder.Decode(&value); err != nil {
+ return nil, &ConnectorValidationError{Message: "REST API response is not valid JSON"}
+ }
+ return value, nil
+}
+
+// newRestAPIPinnedTransport resolves the target once (in assertRestAPIURLSafe)
+// and pins that address for the actual connection, preventing DNS rebinding.
+func newRestAPIPinnedTransport(hostname string, pinIP net.IP) *http.Transport {
+ dialer := &net.Dialer{Timeout: restAPIRequestTimeout, KeepAlive: 30 * time.Second}
+ return &http.Transport{
+ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+ _, port, err := net.SplitHostPort(addr)
+ if err != nil {
+ port = "443"
+ }
+ return dialer.DialContext(ctx, network, net.JoinHostPort(pinIP.String(), port))
+ },
+ TLSClientConfig: &tls.Config{
+ ServerName: hostname,
+ MinVersion: tls.VersionTLS12,
+ },
+ ForceAttemptHTTP2: true,
+ }
+}
+
+// buildURLWithTemplates substitutes {key} placeholders in the URL and returns
+// the remaining query parameters.
+func (c *RestAPIConnector) buildURLWithTemplates(params map[string]any) (string, map[string]any) {
+ u := c.baseURL
+ remaining := make(map[string]any, len(params))
+ for k, v := range params {
+ remaining[k] = v
+ }
+ for k, v := range params {
+ placeholder := "{" + k + "}"
+ if strings.Contains(u, placeholder) {
+ u = strings.ReplaceAll(u, placeholder, restAPIParamString(v))
+ delete(remaining, k)
+ }
+ }
+ return u, remaining
+}
+
+func restAPIParamString(value any) string {
+ switch typed := value.(type) {
+ case string:
+ return typed
+ case json.Number:
+ return typed.String()
+ case bool:
+ if typed {
+ return "True"
+ }
+ return "False"
+ case nil:
+ return ""
+ default:
+ return fmt.Sprint(value)
+ }
+}
+
+func restAPIEncodeParams(params map[string]any) string {
+ if len(params) == 0 {
+ return ""
+ }
+ values := url.Values{}
+ for k, v := range params {
+ values.Set(k, restAPIParamString(v))
+ }
+ return values.Encode()
+}
+
+func restAPINetloc(rawURL string) string {
+ parsed, err := url.Parse(rawURL)
+ if err != nil {
+ return ""
+ }
+ return parsed.Host
+}
+
+func restAPIResolveURL(base, location string) (string, error) {
+ baseParsed, err := url.Parse(base)
+ if err != nil {
+ return "", err
+ }
+ ref, err := url.Parse(location)
+ if err != nil {
+ return "", err
+ }
+ return baseParsed.ResolveReference(ref).String(), nil
+}
+
+func restAPIIsRedirect(status int) bool {
+ switch status {
+ case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,
+ http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
+ return true
+ }
+ return false
+}
+
+var restAPIAuthSensitiveHeaders = map[string]struct{}{
+ "authorization": {},
+ "proxy-authorization": {},
+ "apikey": {},
+ "api-key": {},
+ "x-api-key": {},
+ "x-auth-token": {},
+}
+
+func restAPIStripAuthHeaders(headers map[string]string) map[string]string {
+ out := make(map[string]string, len(headers))
+ for k, v := range headers {
+ if _, sensitive := restAPIAuthSensitiveHeaders[strings.ToLower(k)]; sensitive {
+ continue
+ }
+ out[k] = v
+ }
+ return out
+}
+
+// ---------------------------------------------------------------------------
+// SSRF protection
+// ---------------------------------------------------------------------------
+
+// validateRestAPIURLForSSRF performs quick deny-list checks plus DNS
+// resolution. Resolution failure is logged and tolerated because the
+// per-request check re-validates.
+func validateRestAPIURLForSSRF(rawURL string) error {
+ parsed, err := url.Parse(rawURL)
+ if err != nil {
+ return &ConnectorValidationError{Message: "REST API connector URL must include a hostname."}
+ }
+ if parsed.Scheme != "http" && parsed.Scheme != "https" {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported URL scheme for REST API connector: %q. Only http/https are allowed.", parsed.Scheme)}
+ }
+ hostname := parsed.Hostname()
+ if hostname == "" {
+ return &ConnectorValidationError{Message: "REST API connector URL must include a hostname."}
+ }
+ if strings.EqualFold(hostname, "localhost") {
+ return &ConnectorValidationError{Message: fmt.Sprintf("REST API connector URL hostname %q is not allowed (localhost is blocked).", hostname)}
+ }
+ if restAPISSRFAllowLoopback {
+ return nil
+ }
+ addrs, err := net.LookupIP(hostname)
+ if err != nil {
+ // DNS failure is not an SSRF condition by itself; the per-request
+ // check will surface it if it matters.
+ return nil
+ }
+ for _, addr := range addrs {
+ if !restAPIIPIsGlobal(restAPIEffectiveIP(addr)) {
+ return &ConnectorValidationError{Message: fmt.Sprintf(
+ "REST API connector URL %q resolves to disallowed address %s (localhost, private, link-local, reserved, or multicast addresses are blocked).",
+ rawURL, addr)}
+ }
+ }
+ return nil
+}
+
+// assertRestAPIURLSafe mirrors ssrf_guard.assert_url_is_safe: every resolved
+// address must be globally routable. It returns the hostname and the first
+// validated IP so the caller can pin DNS.
+func assertRestAPIURLSafe(ctx context.Context, rawURL string) (string, net.IP, error) {
+ parsed, err := url.Parse(rawURL)
+ if err != nil {
+ return "", nil, fmt.Errorf("URL is missing a host.")
+ }
+ if parsed.Scheme != "http" && parsed.Scheme != "https" {
+ return "", nil, fmt.Errorf("Disallowed URL scheme: %q. Only [http https] are allowed.", parsed.Scheme)
+ }
+ hostname := parsed.Hostname()
+ if hostname == "" {
+ return "", nil, fmt.Errorf("URL is missing a host.")
+ }
+ if restAPISSRFAllowLoopback {
+ addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)
+ if err != nil {
+ return "", nil, fmt.Errorf("Could not resolve hostname %q: %v", hostname, err)
+ }
+ if len(addrs) == 0 {
+ return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)
+ }
+ return hostname, addrs[0].IP, nil
+ }
+
+ addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)
+ if err != nil {
+ return "", nil, fmt.Errorf("Could not resolve hostname %q: %v", hostname, err)
+ }
+ var first net.IP
+ for _, addr := range addrs {
+ eff := restAPIEffectiveIP(addr.IP)
+ if !restAPIIPIsGlobal(eff) {
+ return "", nil, fmt.Errorf("URL resolves to a non-public address (%s), which is not allowed.", addr.IP)
+ }
+ if first == nil {
+ first = addr.IP
+ }
+ }
+ if first == nil {
+ return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)
+ }
+ return hostname, first, nil
+}
+
+// restAPIEffectiveIP returns the IPv4 equivalent for IPv4-mapped IPv6
+// addresses, mirroring ssrf_guard._effective_ip.
+func restAPIEffectiveIP(ip net.IP) net.IP {
+ if v4 := ip.To4(); v4 != nil && len(ip) == net.IPv6len {
+ return v4
+ }
+ return ip
+}
+
+// restAPIIPIsGlobal mirrors ipaddress.is_global for the address classes that
+// matter in practice.
+func restAPIIPIsGlobal(ip net.IP) bool {
+ if ip == nil {
+ return false
+ }
+ if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() || ip.IsMulticast() ||
+ ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() {
+ return false
+ }
+ if v4 := ip.To4(); v4 != nil {
+ first := v4[0]
+ // 0.0.0.0/8, 100.64.0.0/10 (CGNAT), 192.0.0.0/24 (IETF protocol
+ // assignments), 198.18.0.0/15 (benchmarking), 240.0.0.0/4, broadcast.
+ if first == 0 || first == 100 || (first == 192 && v4[1] == 0) ||
+ (first == 198 && v4[1]&0xfe == 18) || first >= 240 {
+ return false
+ }
+ return true
+ }
+ // IPv6 documentation range (2001:db8::/32) is not globally routable.
+ if len(ip) == net.IPv6len && ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x0d && ip[3] == 0xb8 {
+ return false
+ }
+ return true
+}
+
+// ---------------------------------------------------------------------------
+// Pagination
+// ---------------------------------------------------------------------------
+
+// resolvePageSize mirrors _resolve_page_size.
+func (c *RestAPIConnector) resolvePageSize() (int, error) {
+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["page_size"]); ok && v > 0 {
+ return v, nil
+ }
+ sizeParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["page_size_param"]))
+ if sizeParam == "" {
+ sizeParam = strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["limit_param"]))
+ }
+ if sizeParam != "" {
+ for _, source := range []map[string]string{c.cfg.QueryParams, c.urlParams} {
+ if raw, ok := source[sizeParam]; ok {
+ if v, err := strconv.Atoi(raw); err == nil && v > 0 {
+ return v, nil
+ }
+ }
+ }
+ }
+ return c.cfg.BatchSize, nil
+}
+
+func (c *RestAPIConnector) applyPagePagination(params map[string]any, page, perPage int) {
+ pageParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["page_param"]))
+ if pageParam == "" {
+ pageParam = "page"
+ }
+ params[pageParam] = page
+ if sizeParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["page_size_param"])); sizeParam != "" {
+ params[sizeParam] = perPage
+ }
+}
+
+func (c *RestAPIConnector) applyOffsetPagination(params map[string]any, offset, limit int) {
+ offsetParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["offset_param"]))
+ if offsetParam == "" {
+ offsetParam = "offset"
+ }
+ params[offsetParam] = offset
+ if limitParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["limit_param"])); limitParam != "" {
+ params[limitParam] = limit
+ }
+}
+
+func (c *RestAPIConnector) applyCursorPagination(params map[string]any, cursor string) {
+ cursorParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["cursor_param"]))
+ if cursorParam == "" {
+ cursorParam = "cursor"
+ }
+ params[cursorParam] = cursor
+}
+
+// restAPIItemIterator mirrors _iter_items: it walks pages applying the
+// configured pagination and stopping when the source reports no more items.
+type restAPIItemIterator struct {
+ c *RestAPIConnector
+ pageCount int
+ page int
+ offset int
+ limit int
+ perPage int
+ cursor string
+ finished bool
+}
+
+func newRestAPIItemIterator(c *RestAPIConnector) *restAPIItemIterator {
+ it := &restAPIItemIterator{c: c}
+ if perPage, err := c.resolvePageSize(); err == nil {
+ it.perPage = perPage
+ } else {
+ it.perPage = c.cfg.BatchSize
+ }
+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["start_page"]); ok {
+ it.page = v
+ } else {
+ it.page = 1
+ }
+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["start_offset"]); ok {
+ it.offset = v
+ }
+ it.limit = it.perPage
+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["limit"]); ok {
+ it.limit = v
+ }
+ if it.limit <= 0 {
+ it.limit = it.perPage
+ }
+ it.cursor = strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["initial_cursor"]))
+ return it
+}
+
+// nextPage returns the items of the next page, or nil when iteration is done.
+func (it *restAPIItemIterator) nextPage(ctx context.Context) ([]map[string]any, error) {
+ if it.finished || it.pageCount >= it.c.cfg.MaxPages {
+ return nil, nil
+ }
+ params := map[string]any{}
+ switch it.c.cfg.PaginationType {
+ case restAPIPaginationPage:
+ it.c.applyPagePagination(params, it.page, it.perPage)
+ case restAPIPaginationOffset:
+ it.c.applyOffsetPagination(params, it.offset, it.limit)
+ case restAPIPaginationCursor:
+ if it.cursor != "" {
+ it.c.applyCursorPagination(params, it.cursor)
+ }
+ }
+ if it.pageCount > 0 && it.c.cfg.RequestDelay > 0 {
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-time.After(time.Duration(it.c.cfg.RequestDelay * float64(time.Second))):
+ }
+ }
+
+ response, err := it.c.fetchPage(ctx, params)
+ if err != nil {
+ var (
+ valErr *ConnectorValidationError
+ credErr *ConnectorMissingCredentialError
+ )
+ if errors.As(err, &valErr) || errors.As(err, &credErr) {
+ return nil, err
+ }
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("REST API page fetch failed: %v", err)}
+ }
+
+ items := restAPIExtractItems(response, it.c.cfg.ItemsPath)
+ if len(items) == 0 {
+ it.finished = true
+ return nil, nil
+ }
+ it.pageCount++
+
+ switch it.c.cfg.PaginationType {
+ case restAPIPaginationNone:
+ it.finished = true
+ case restAPIPaginationPage:
+ if len(items) < it.perPage {
+ it.finished = true
+ } else {
+ it.page++
+ }
+ case restAPIPaginationOffset:
+ if len(items) < it.limit {
+ it.finished = true
+ } else {
+ it.offset += it.limit
+ }
+ case restAPIPaginationCursor:
+ next := restAPIExtractNextCursor(response, it.c.cfg.PaginationConfig)
+ if next == "" {
+ it.finished = true
+ } else {
+ it.cursor = next
+ }
+ }
+ return items, nil
+}
+
+// positionAfterPage returns the resume position after the page that was just
+// consumed, i.e. the next page to fetch. It is nil for pagination_type=none,
+// which never emits checkpoints.
+func (it *restAPIItemIterator) positionAfterPage() *restAPISyncCursor {
+ switch it.c.cfg.PaginationType {
+ case restAPIPaginationPage:
+ return &restAPISyncCursor{Page: it.page}
+ case restAPIPaginationOffset:
+ return &restAPISyncCursor{Offset: it.offset}
+ case restAPIPaginationCursor:
+ return &restAPISyncCursor{Cursor: it.cursor}
+ }
+ return nil
+}
+
+// ---------------------------------------------------------------------------
+// JSON extraction
+// ---------------------------------------------------------------------------
+
+// restAPIExtractItems mirrors _extract_items.
+func restAPIExtractItems(response any, itemsPath string) []map[string]any {
+ var items []any
+ if itemsPath != "" {
+ matches, err := restAPIJSONPathMatches(response, itemsPath)
+ if err != nil {
+ return nil
+ }
+ if len(matches) == 0 {
+ return nil
+ }
+ if len(matches) == 1 {
+ if list, ok := matches[0].([]any); ok {
+ items = list
+ } else {
+ items = matches
+ }
+ } else {
+ items = matches
+ }
+ } else if list, ok := response.([]any); ok {
+ items = list
+ } else if dict, ok := response.(map[string]any); ok {
+ for _, key := range []string{"items", "results", "data", "records"} {
+ if value, ok := dict[key]; ok {
+ if list, ok := value.([]any); ok {
+ items = list
+ break
+ }
+ }
+ }
+ if items == nil {
+ for _, value := range dict {
+ if list, ok := value.([]any); ok {
+ items = list
+ break
+ }
+ }
+ }
+ }
+
+ out := make([]map[string]any, 0, len(items))
+ for _, item := range items {
+ if mapping, ok := item.(map[string]any); ok {
+ out = append(out, mapping)
+ }
+ }
+ return out
+}
+
+// restAPIExtractNextCursor mirrors _extract_next_cursor.
+func restAPIExtractNextCursor(response any, paginationConfig map[string]any) string {
+ cursorPath := strings.TrimSpace(stringConfig(paginationConfig["next_cursor_path"]))
+ if cursorPath == "" {
+ field := strings.TrimSpace(stringConfig(paginationConfig["next_cursor_field"]))
+ if field != "" {
+ if dict, ok := response.(map[string]any); ok {
+ if value, ok := dict[field]; ok && value != nil {
+ return fmt.Sprint(value)
+ }
+ }
+ }
+ return ""
+ }
+ matches, err := restAPIJSONPathMatches(response, cursorPath)
+ if err != nil || len(matches) == 0 || matches[0] == nil {
+ return ""
+ }
+ return fmt.Sprint(matches[0])
+}
+
+const (
+ restAPIJSONPathField = iota
+ restAPIJSONPathIndex
+ restAPIJSONPathWildcard
+)
+
+type restAPIJSONPathToken struct {
+ kind int
+ key string
+ index int
+ recursive bool
+}
+
+// restAPIJSONPathMatches implements the commonly used subset of jsonpath: $,
+// dot keys, [n], [*], and .. recursive descent.
+func restAPIJSONPathMatches(root any, path string) ([]any, error) {
+ tokens, err := parseRestAPIJSONPath(path)
+ if err != nil {
+ return nil, err
+ }
+ current := []any{root}
+ for _, token := range tokens {
+ next := []any{}
+ for _, value := range current {
+ if token.recursive {
+ restAPIJSONPathCollectRecursive(value, token, &next)
+ continue
+ }
+ switch token.kind {
+ case restAPIJSONPathField:
+ if dict, ok := value.(map[string]any); ok {
+ if child, ok := dict[token.key]; ok {
+ next = append(next, child)
+ }
+ }
+ case restAPIJSONPathIndex:
+ if list, ok := value.([]any); ok && token.index >= 0 && token.index < len(list) {
+ next = append(next, list[token.index])
+ }
+ case restAPIJSONPathWildcard:
+ switch typed := value.(type) {
+ case []any:
+ next = append(next, typed...)
+ case map[string]any:
+ for _, child := range typed {
+ next = append(next, child)
+ }
+ }
+ }
+ }
+ current = next
+ if len(current) == 0 {
+ break
+ }
+ }
+ return current, nil
+}
+
+func restAPIJSONPathCollectRecursive(value any, token restAPIJSONPathToken, out *[]any) {
+ if token.kind != restAPIJSONPathField {
+ return
+ }
+ if dict, ok := value.(map[string]any); ok {
+ if child, ok := dict[token.key]; ok {
+ *out = append(*out, child)
+ }
+ for _, nested := range dict {
+ restAPIJSONPathCollectRecursive(nested, token, out)
+ }
+ }
+ if list, ok := value.([]any); ok {
+ for _, nested := range list {
+ restAPIJSONPathCollectRecursive(nested, token, out)
+ }
+ }
+}
+
+func parseRestAPIJSONPath(path string) ([]restAPIJSONPathToken, error) {
+ s := strings.TrimSpace(path)
+ if strings.HasPrefix(s, "$") {
+ s = s[1:]
+ }
+ var tokens []restAPIJSONPathToken
+ pendingRecursive := false
+ i := 0
+ for i < len(s) {
+ switch s[i] {
+ case '.':
+ if i+1 < len(s) && s[i+1] == '.' {
+ pendingRecursive = true
+ i += 2
+ } else {
+ i++
+ }
+ case '[':
+ end := strings.IndexByte(s[i:], ']')
+ if end < 0 {
+ return nil, fmt.Errorf("unterminated bracket in JSONPath")
+ }
+ content := strings.TrimSpace(s[i+1 : i+end])
+ token := restAPIJSONPathToken{recursive: pendingRecursive}
+ pendingRecursive = false
+ switch {
+ case content == "*":
+ token.kind = restAPIJSONPathWildcard
+ default:
+ if n, err := strconv.Atoi(content); err == nil {
+ token.kind = restAPIJSONPathIndex
+ token.index = n
+ } else {
+ token.kind = restAPIJSONPathField
+ token.key = strings.Trim(content, "'\"")
+ }
+ }
+ tokens = append(tokens, token)
+ i += end + 1
+ default:
+ start := i
+ for i < len(s) && s[i] != '.' && s[i] != '[' {
+ i++
+ }
+ key := s[start:i]
+ if key == "" {
+ return nil, fmt.Errorf("empty JSONPath segment")
+ }
+ tokens = append(tokens, restAPIJSONPathToken{kind: restAPIJSONPathField, key: key, recursive: pendingRecursive})
+ pendingRecursive = false
+ }
+ }
+ return tokens, nil
+}
+
+// ---------------------------------------------------------------------------
+// Field extraction and item-to-document mapping
+// ---------------------------------------------------------------------------
+
+var restAPIFieldSegmentRE = regexp.MustCompile(`^([^\[\]]+)(\[(\d+|\*)\])?$`)
+
+// extractRestAPIFieldValues mirrors _extract_field_values: dot-notation with
+// optional [index] / [*] segments.
+func extractRestAPIFieldValues(item map[string]any, path string) []any {
+ if path == "" {
+ return nil
+ }
+ current := []any{item}
+ for _, segment := range strings.Split(path, ".") {
+ if segment == "" {
+ return nil
+ }
+ match := restAPIFieldSegmentRE.FindStringSubmatch(segment)
+ key := segment
+ index := ""
+ if match != nil {
+ key = match[1]
+ index = match[3]
+ }
+ next := []any{}
+ for _, value := range current {
+ dict, ok := value.(map[string]any)
+ if !ok {
+ continue
+ }
+ child, ok := dict[key]
+ if !ok || child == nil {
+ continue
+ }
+ if index == "" {
+ next = append(next, child)
+ continue
+ }
+ list, ok := child.([]any)
+ if !ok {
+ continue
+ }
+ if index == "*" {
+ next = append(next, list...)
+ continue
+ }
+ idx, err := strconv.Atoi(index)
+ if err != nil {
+ continue
+ }
+ if idx >= 0 && idx < len(list) {
+ next = append(next, list[idx])
+ }
+ }
+ current = next
+ if len(current) == 0 {
+ break
+ }
+ }
+ return current
+}
+
+func extractRestAPIField(item map[string]any, path string) any {
+ values := extractRestAPIFieldValues(item, path)
+ if len(values) == 0 {
+ return nil
+ }
+ if len(values) == 1 {
+ return values[0]
+ }
+ return values
+}
+
+// getTypedFieldValue mirrors _get_typed_field_value.
+func (c *RestAPIConnector) getTypedFieldValue(path string, item map[string]any) any {
+ values := extractRestAPIFieldValues(item, path)
+ if len(values) == 0 {
+ return c.cfg.FieldDefaultValues[path]
+ }
+ hint := c.cfg.FieldTypeHints[path]
+ converted := make([]any, 0, len(values))
+ for _, value := range values {
+ if cv := convertRestAPITypedValue(value, hint); cv != nil {
+ converted = append(converted, cv)
+ }
+ }
+ if len(converted) == 0 {
+ return nil
+ }
+ if len(converted) == 1 {
+ return converted[0]
+ }
+ parts := make([]string, 0, len(converted))
+ for _, value := range converted {
+ parts = append(parts, coerceRestAPIToText(value))
+ }
+ return strings.Join(parts, ", ")
+}
+
+func convertRestAPITypedValue(value any, hint string) any {
+ switch hint {
+ case "string":
+ return restAPIValueText(value)
+ case "number":
+ f, ok := restAPIToFloat(value)
+ if !ok {
+ return nil
+ }
+ if f == math.Trunc(f) && !math.IsInf(f, 0) && math.Abs(f) < 1e15 {
+ return int64(f)
+ }
+ return f
+ case "date":
+ if t, ok := value.(time.Time); ok {
+ return restAPITimeISO(t)
+ }
+ if t := parseRestAPIDatetime(value); t != nil {
+ return restAPITimeISO(*t)
+ }
+ return restAPIValueText(value)
+ default:
+ return value
+ }
+}
+
+func restAPIToFloat(value any) (float64, bool) {
+ switch typed := value.(type) {
+ case json.Number:
+ f, err := typed.Float64()
+ return f, err == nil
+ case float64:
+ return typed, true
+ case int:
+ return float64(typed), true
+ case int64:
+ return float64(typed), true
+ case string:
+ f, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
+ return f, err == nil
+ default:
+ return 0, false
+ }
+}
+
+// extractTimestamp mirrors _extract_timestamp.
+func (c *RestAPIConnector) extractTimestamp(item map[string]any) *time.Time {
+ if c.cfg.PollTimestampField == "" {
+ return nil
+ }
+ value := extractRestAPIField(item, c.cfg.PollTimestampField)
+ if list, ok := value.([]any); ok && len(list) > 0 {
+ value = list[0]
+ }
+ return parseRestAPIDatetime(value)
+}
+
+// parseRestAPIDatetime mirrors _parse_datetime.
+func parseRestAPIDatetime(value any) *time.Time {
+ if value == nil {
+ return nil
+ }
+ switch typed := value.(type) {
+ case time.Time:
+ t := typed.UTC()
+ return &t
+ case json.Number:
+ if f, err := typed.Float64(); err == nil {
+ return restAPITimestampFromFloat(f)
+ }
+ case float64:
+ return restAPITimestampFromFloat(typed)
+ case int:
+ return restAPITimestampFromFloat(float64(typed))
+ case int64:
+ return restAPITimestampFromFloat(float64(typed))
+ case string:
+ text := strings.TrimSpace(typed)
+ if text == "" {
+ return nil
+ }
+ layouts := []string{
+ "2006-01-02T15:04:05.999999999Z",
+ "2006-01-02T15:04:05Z",
+ "2006-01-02 15:04:05",
+ "2006-01-02",
+ }
+ for _, layout := range layouts {
+ if t, err := time.Parse(layout, text); err == nil {
+ t = t.UTC()
+ return &t
+ }
+ }
+ normalized := strings.ReplaceAll(text, " ", "T")
+ if strings.HasSuffix(normalized, "Z") {
+ normalized = normalized[:len(normalized)-1] + "+00:00"
+ }
+ for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05.999999999-07:00", "2006-01-02T15:04:05-07:00"} {
+ if t, err := time.Parse(layout, normalized); err == nil {
+ t = t.UTC()
+ return &t
+ }
+ }
+ }
+ return nil
+}
+
+func restAPITimestampFromFloat(f float64) *time.Time {
+ sec := int64(f)
+ nsec := int64((f - float64(sec)) * 1e9)
+ t := time.Unix(sec, nsec).UTC()
+ return &t
+}
+
+// itemToDocument mirrors _item_to_document.
+func (c *RestAPIConnector) itemToDocument(item map[string]any) (SourceDocument, error) {
+ var rawID string
+ if c.cfg.IDField != "" {
+ if v := c.getTypedFieldValue(c.cfg.IDField, item); v != nil {
+ rawID = coerceRestAPIToText(v)
+ } else {
+ rawID = restAPIHash128("rest_api_item:" + restAPIStableText(item))
+ }
+ } else {
+ rawID = restAPIHash128("rest_api_item:" + restAPIStableText(item))
+ }
+ docID := restAPIHash128("rest_api:" + rawID)
+
+ var contentText string
+ if c.cfg.ContentTemplate != "" {
+ contentText = c.renderContentTemplate(item)
+ } else {
+ parts := []string{}
+ for _, field := range c.cfg.ContentFields {
+ if v := c.getTypedFieldValue(field, item); v != nil {
+ if text := restAPIStripHTML(coerceRestAPIToText(v)); text != "" {
+ parts = append(parts, text)
+ }
+ }
+ }
+ contentText = strings.Join(parts, "\n")
+ }
+ blob := []byte(contentText)
+
+ var metadata map[string]any
+ if len(c.cfg.MetadataFields) > 0 {
+ metadata = map[string]any{}
+ for _, field := range c.cfg.MetadataFields {
+ if v := c.getTypedFieldValue(field, item); v != nil {
+ metadata[field] = serializeRestAPIMetadataValue(v)
+ }
+ }
+ }
+
+ updatedAt := c.extractTimestamp(item)
+ if updatedAt == nil {
+ now := time.Now().UTC()
+ updatedAt = &now
+ }
+
+ sem := rawID
+ if len(c.cfg.ContentFields) > 0 {
+ sem = restAPIValueText(extractRestAPIField(item, c.cfg.ContentFields[0]))
+ }
+ sem = restAPIStripHTML(sem)
+ sem = strings.ReplaceAll(sem, "\n", " ")
+ sem = strings.ReplaceAll(sem, "\r", " ")
+ sem = strings.TrimSpace(sem)
+ if len(sem) > 100 {
+ sem = sem[:100]
+ }
+ if sem == "" {
+ sem = docID
+ }
+
+ return SourceDocument{
+ SourceID: docID,
+ SemanticIdentifier: sem,
+ Extension: ".txt",
+ Blob: blob,
+ UpdatedAt: *updatedAt,
+ SizeBytes: int64(len(blob)),
+ Metadata: metadata,
+ Fingerprint: contentFingerprint(blob),
+ }, nil
+}
+
+// renderContentTemplate mirrors _render_content_template.
+func (c *RestAPIConnector) renderContentTemplate(item map[string]any) string {
+ values := map[string]string{}
+ seen := map[string]bool{}
+ fields := append(append([]string{}, c.cfg.ContentFields...), c.cfg.MetadataFields...)
+ for _, field := range fields {
+ if seen[field] {
+ continue
+ }
+ seen[field] = true
+ if v := c.getTypedFieldValue(field, item); v != nil {
+ values[restAPIFieldToTemplateName(field)] = coerceRestAPIToText(v)
+ }
+ }
+ rendered, ok := renderRestAPITemplateSafeMap(c.cfg.ContentTemplate, values)
+ if !ok {
+ parts := []string{}
+ for _, field := range c.cfg.ContentFields {
+ if p := coerceRestAPIToText(c.getTypedFieldValue(field, item)); p != "" {
+ parts = append(parts, p)
+ }
+ }
+ rendered = strings.Join(parts, "\n")
+ }
+ return restAPIStripHTML(rendered)
+}
+
+var restAPITemplateIndexRE = regexp.MustCompile(`\[\d+\]|\[\*\]`)
+
+func restAPIFieldToTemplateName(field string) string {
+ name := restAPITemplateIndexRE.ReplaceAllString(field, "")
+ return strings.ReplaceAll(name, ".", "_")
+}
+
+// renderRestAPITemplateSafeMap renders {name} placeholders from values with a
+// safe lookup: unknown names become the empty string; malformed placeholders
+// make the whole render fail so the caller falls back to joined content.
+func renderRestAPITemplateSafeMap(template string, values map[string]string) (string, bool) {
+ var b strings.Builder
+ for i := 0; i < len(template); {
+ switch template[i] {
+ case '{':
+ if i+1 < len(template) && template[i+1] == '{' {
+ b.WriteByte('{')
+ i += 2
+ continue
+ }
+ end := strings.IndexByte(template[i:], '}')
+ if end < 0 {
+ return "", false
+ }
+ token := template[i+1 : i+end]
+ if !validRestAPITemplateKey(token) {
+ return "", false
+ }
+ b.WriteString(values[token])
+ i += end + 1
+ case '}':
+ if i+1 < len(template) && template[i+1] == '}' {
+ b.WriteByte('}')
+ i += 2
+ continue
+ }
+ return "", false
+ default:
+ b.WriteByte(template[i])
+ i++
+ }
+ }
+ return b.String(), true
+}
+
+func validRestAPITemplateKey(token string) bool {
+ if token == "" {
+ return false
+ }
+ for _, r := range token {
+ if r != '_' && !unicode.IsLetter(r) && !unicode.IsDigit(r) {
+ return false
+ }
+ }
+ return true
+}
+
+// restAPIHash128 mirrors api.utils.common.hash128 (xxhash128 hex digest).
+func restAPIHash128(data string) string {
+ return contentFingerprint([]byte(data))
+}
+
+// restAPIStableText renders a JSON value as deterministic text: single quotes,
+// True/False/None, and matching float formatting. Map keys are sorted to keep
+// the fallback document ID stable across runs.
+func restAPIStableText(value any) string {
+ switch typed := value.(type) {
+ case nil:
+ return "None"
+ case bool:
+ if typed {
+ return "True"
+ }
+ return "False"
+ case string:
+ return restAPIQuoteString(typed)
+ case json.Number:
+ if strings.ContainsAny(typed.String(), ".eE") {
+ if f, err := typed.Float64(); err == nil {
+ return restAPIFloatText(f)
+ }
+ }
+ return typed.String()
+ case int:
+ return strconv.Itoa(typed)
+ case int64:
+ return strconv.FormatInt(typed, 10)
+ case float64:
+ return restAPIFloatText(typed)
+ case []any:
+ parts := make([]string, 0, len(typed))
+ for _, item := range typed {
+ parts = append(parts, restAPIStableText(item))
+ }
+ return "[" + strings.Join(parts, ", ") + "]"
+ case map[string]any:
+ keys := make([]string, 0, len(typed))
+ for k := range typed {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ parts := make([]string, 0, len(keys))
+ for _, k := range keys {
+ parts = append(parts, restAPIQuoteString(k)+": "+restAPIStableText(typed[k]))
+ }
+ return "{" + strings.Join(parts, ", ") + "}"
+ default:
+ return fmt.Sprint(value)
+ }
+}
+
+// restAPIQuoteString renders a string as a single-quoted literal with the
+// standard backslash escapes; printable non-ASCII characters are kept as-is.
+func restAPIQuoteString(s string) string {
+ var b strings.Builder
+ b.WriteByte('\'')
+ for _, r := range s {
+ switch r {
+ case '\\':
+ b.WriteString(`\\`)
+ case '\'':
+ b.WriteString(`\'`)
+ case '\n':
+ b.WriteString(`\n`)
+ case '\r':
+ b.WriteString(`\r`)
+ case '\t':
+ b.WriteString(`\t`)
+ case '\b':
+ b.WriteString(`\b`)
+ case '\f':
+ b.WriteString(`\f`)
+ default:
+ if r < 0x20 || r == 0x7f {
+ if r < 0x100 {
+ b.WriteString(fmt.Sprintf(`\x%02x`, r))
+ } else {
+ b.WriteString(fmt.Sprintf(`\u%04x`, r))
+ }
+ } else {
+ b.WriteRune(r)
+ }
+ }
+ }
+ b.WriteByte('\'')
+ return b.String()
+}
+
+// restAPIValueText renders the scalar and composite JSON values the connector deals with
+// as plain text.
+func restAPIValueText(value any) string {
+ switch typed := value.(type) {
+ case nil:
+ return "None"
+ case bool:
+ if typed {
+ return "True"
+ }
+ return "False"
+ case string:
+ return typed
+ case json.Number:
+ if strings.ContainsAny(typed.String(), ".eE") {
+ if f, err := typed.Float64(); err == nil {
+ return restAPIFloatText(f)
+ }
+ }
+ return typed.String()
+ case int:
+ return strconv.Itoa(typed)
+ case int64:
+ return strconv.FormatInt(typed, 10)
+ case float64:
+ return restAPIFloatText(typed)
+ default:
+ return restAPIStableText(value)
+ }
+}
+
+// restAPIFloatText renders floats in a stable textual form.
+func restAPIFloatText(f float64) string {
+ switch {
+ case math.IsInf(f, 1):
+ return "inf"
+ case math.IsInf(f, -1):
+ return "-inf"
+ case math.IsNaN(f):
+ return "nan"
+ case f == math.Trunc(f) && math.Abs(f) < 1e16:
+ return strconv.FormatFloat(f, 'f', 1, 64)
+ default:
+ return strconv.FormatFloat(f, 'g', -1, 64)
+ }
+}
+
+// coerceRestAPIToText mirrors _coerce_to_text.
+func coerceRestAPIToText(value any) string {
+ switch typed := value.(type) {
+ case nil:
+ return ""
+ case string:
+ return typed
+ case json.Number:
+ if strings.ContainsAny(typed.String(), ".eE") {
+ if f, err := typed.Float64(); err == nil {
+ return restAPIFloatText(f)
+ }
+ }
+ return typed.String()
+ case int:
+ return strconv.Itoa(typed)
+ case int64:
+ return strconv.FormatInt(typed, 10)
+ case float64:
+ return restAPIFloatText(typed)
+ case bool:
+ if typed {
+ return "True"
+ }
+ return "False"
+ default:
+ return pyJSONDumps(value)
+ }
+}
+
+// serializeRestAPIMetadataValue mirrors _serialize_metadata_value.
+func serializeRestAPIMetadataValue(value any) any {
+ switch typed := value.(type) {
+ case time.Time:
+ return restAPITimeISO(typed)
+ case json.Number:
+ if f, err := typed.Float64(); err == nil {
+ if f == math.Trunc(f) && !math.IsInf(f, 0) && math.Abs(f) < 1e15 {
+ return int64(f)
+ }
+ return f
+ }
+ return typed.String()
+ case int, int64, float64, bool, string:
+ return value
+ default:
+ return pyJSONDumps(value)
+ }
+}
+
+// pyJSONDumps mirrors json.dumps(value, ensure_ascii=False) with the default
+// separators (", " and ": ").
+func pyJSONDumps(value any) string {
+ var b strings.Builder
+ pyJSONDumpsTo(&b, value)
+ return b.String()
+}
+
+func pyJSONDumpsTo(b *strings.Builder, value any) {
+ switch typed := value.(type) {
+ case nil:
+ b.WriteString("null")
+ case bool:
+ if typed {
+ b.WriteString("true")
+ } else {
+ b.WriteString("false")
+ }
+ case string:
+ raw, _ := json.Marshal(typed)
+ b.Write(raw)
+ case json.Number:
+ if strings.ContainsAny(typed.String(), ".eE") {
+ if f, err := typed.Float64(); err == nil {
+ b.WriteString(restAPIFloatText(f))
+ return
+ }
+ }
+ b.WriteString(typed.String())
+ case int:
+ b.WriteString(strconv.Itoa(typed))
+ case int64:
+ b.WriteString(strconv.FormatInt(typed, 10))
+ case float64:
+ b.WriteString(restAPIFloatText(typed))
+ case []any:
+ b.WriteByte('[')
+ for i, item := range typed {
+ if i > 0 {
+ b.WriteString(", ")
+ }
+ pyJSONDumpsTo(b, item)
+ }
+ b.WriteByte(']')
+ case map[string]any:
+ keys := make([]string, 0, len(typed))
+ for k := range typed {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ b.WriteByte('{')
+ for i, k := range keys {
+ if i > 0 {
+ b.WriteString(", ")
+ }
+ raw, _ := json.Marshal(k)
+ b.Write(raw)
+ b.WriteString(": ")
+ pyJSONDumpsTo(b, typed[k])
+ }
+ b.WriteByte('}')
+ default:
+ b.WriteString(fmt.Sprint(value))
+ }
+}
+
+var restAPITagRE = regexp.MustCompile(`<[^>]+>`)
+var restAPIWhitespaceRE = regexp.MustCompile(`\s+`)
+
+// restAPIStripHTML mirrors _strip_html.
+func restAPIStripHTML(text string) string {
+ if !strings.Contains(text, "<") || !strings.Contains(text, ">") {
+ return text
+ }
+ cleaned := restAPITagRE.ReplaceAllString(text, " ")
+ return strings.TrimSpace(restAPIWhitespaceRE.ReplaceAllString(cleaned, " "))
+}
+
+// restAPITimeISO renders a UTC time as an ISO 8601 string, keeping fractional
+// seconds when non-zero.
+func restAPITimeISO(t time.Time) string {
+ t = t.UTC()
+ base := t.Format("2006-01-02T15:04:05")
+ if t.Nanosecond() != 0 {
+ base += fmt.Sprintf(".%06d", t.Nanosecond()/1000)
+ }
+ return base + "+00:00"
+}
+
+// ---------------------------------------------------------------------------
+// Sync session
+// ---------------------------------------------------------------------------
+
+// restAPISyncCursor is the page-boundary resume position serialized into
+// SyncCheckpoint.Cursor. It stores the next page to fetch for the configured
+// pagination type; pagination_type=none never produces a checkpoint.
+type restAPISyncCursor struct {
+ Page int `json:"page,omitempty"`
+ Offset int `json:"offset,omitempty"`
+ Cursor string `json:"cursor,omitempty"`
+}
+
+// restAPIPageState tracks one fetched page until all of its documents have
+// been handed out in emitted batches. nextPos is the resume position after the
+// page (nil for pagination_type=none).
+type restAPIPageState struct {
+ remaining int
+ nextPos *restAPISyncCursor
+}
+
+// restAPISyncSession streams items lazily so large APIs are not fully
+// materialized in memory.
+type restAPISyncSession struct {
+ connector *RestAPIConnector
+ iterator *restAPIItemIterator
+ batchSize int
+ fromBeginning bool
+ windowStart *time.Time
+ windowEnd time.Time
+ pending []SourceDocument
+ pages []restAPIPageState
+ drained *restAPISyncCursor
+}
+
+// applyResume restores the next page to fetch from a saved checkpoint. Invalid
+// or empty cursors are ignored, mirroring the tolerant resume of other
+// connectors.
+func (s *restAPISyncSession) applyResume(checkpoint *SyncCheckpoint) {
+ if checkpoint == nil || checkpoint.Cursor == "" {
+ return
+ }
+ var cursor restAPISyncCursor
+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
+ return
+ }
+ switch s.connector.cfg.PaginationType {
+ case restAPIPaginationPage:
+ if cursor.Page > 0 {
+ s.iterator.page = cursor.Page
+ }
+ case restAPIPaginationOffset:
+ if cursor.Offset > 0 {
+ s.iterator.offset = cursor.Offset
+ }
+ case restAPIPaginationCursor:
+ if cursor.Cursor != "" {
+ s.iterator.cursor = cursor.Cursor
+ }
+ }
+}
+
+// NextBatch returns the next batch of source documents. The batch carries a
+// checkpoint only when at least one page has been fully handed out: the
+// checkpoint points at the next page to fetch, so a retried task re-fetches at
+// most the page where the last batch ended.
+func (s *restAPISyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ for len(s.pending) < s.batchSize {
+ items, err := s.iterator.nextPage(ctx)
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ if items == nil {
+ break
+ }
+ pageDocs := 0
+ for _, item := range items {
+ doc, err := s.connector.itemToDocument(item)
+ if err != nil {
+ // Items that fail conversion are skipped with a warning.
+ continue
+ }
+ if !s.fromBeginning && s.windowStart != nil {
+ if !restAPIDocInTimeWindow(doc.UpdatedAt, *s.windowStart, s.windowEnd) {
+ continue
+ }
+ }
+ s.pending = append(s.pending, doc)
+ pageDocs++
+ }
+ s.pages = append(s.pages, restAPIPageState{remaining: pageDocs, nextPos: s.iterator.positionAfterPage()})
+ }
+ if len(s.pending) == 0 {
+ return SyncBatch{}, io.EOF
+ }
+ end := s.batchSize
+ if end > len(s.pending) {
+ end = len(s.pending)
+ }
+ batch := SyncBatch{Documents: s.pending[:end]}
+ s.pending = s.pending[end:]
+
+ // Advance the resume point past every page fully covered by the batches
+ // emitted so far. A page whose tail is still pending keeps the checkpoint
+ // at the page start, so a retry re-fetches at most that one page.
+ for len(s.pages) > 0 {
+ if s.pages[0].remaining > end {
+ s.pages[0].remaining -= end
+ break
+ }
+ end -= s.pages[0].remaining
+ if s.pages[0].nextPos != nil {
+ s.drained = s.pages[0].nextPos
+ }
+ s.pages = s.pages[1:]
+ if end == 0 {
+ break
+ }
+ }
+ if s.drained != nil {
+ if raw, err := json.Marshal(s.drained); err == nil {
+ batch.Checkpoint = &SyncCheckpoint{Cursor: string(raw)}
+ }
+ }
+ return batch, nil
+}
+
+// Close releases the session.
+func (s *restAPISyncSession) Close() error {
+ return nil
+}
+
+// restAPIDocInTimeWindow mirrors _doc_in_time_window: start <= dt < end.
+func restAPIDocInTimeWindow(docUpdatedAt, start, end time.Time) bool {
+ dt := docUpdatedAt.UTC()
+ return !dt.Before(start) && dt.Before(end)
+}
diff --git a/internal/syncer/connector/rest_api_test.go b/internal/syncer/connector/rest_api_test.go
new file mode 100644
index 0000000000..b0777d8fe4
--- /dev/null
+++ b/internal/syncer/connector/rest_api_test.go
@@ -0,0 +1,1240 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func withRestAPITestHooks(t *testing.T) {
+ t.Helper()
+ origLoopback := restAPISSRFAllowLoopback
+ origTries := restAPIRetryTries
+ origBaseDelay := restAPIRetryBaseDelay
+ origMaxDelay := restAPIRetryMaxDelay
+ origBackoff := restAPIRetryBackoff
+ origJitter := restAPIRetryJitter
+ orig429Waits := restAPI429MaxWaits
+ orig429Wait := restAPI429DefaultWait
+ restAPISSRFAllowLoopback = true
+ restAPIRetryTries = 3
+ restAPIRetryBaseDelay = time.Millisecond
+ restAPIRetryMaxDelay = 10 * time.Millisecond
+ restAPIRetryBackoff = 2
+ restAPIRetryJitter = 0
+ restAPI429MaxWaits = 3
+ restAPI429DefaultWait = time.Millisecond
+ t.Cleanup(func() {
+ restAPISSRFAllowLoopback = origLoopback
+ restAPIRetryTries = origTries
+ restAPIRetryBaseDelay = origBaseDelay
+ restAPIRetryMaxDelay = origMaxDelay
+ restAPIRetryBackoff = origBackoff
+ restAPIRetryJitter = origJitter
+ restAPI429MaxWaits = orig429Waits
+ restAPI429DefaultWait = orig429Wait
+ })
+}
+
+func mustRestAPIConnector(t *testing.T, config map[string]any) *RestAPIConnector {
+ t.Helper()
+ c, err := NewRestAPIConnector(config)
+ if err != nil {
+ t.Fatalf("NewRestAPIConnector: %v", err)
+ }
+ return c
+}
+
+func TestNewRestAPIConnectorDefaults(t *testing.T) {
+ withRestAPITestHooks(t)
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": "https://example.com/api",
+ "content_fields": "title, body",
+ })
+ if c.cfg.Method != "GET" {
+ t.Fatalf("method=%q want GET", c.cfg.Method)
+ }
+ if c.cfg.BatchSize != 2 {
+ t.Fatalf("batch_size=%d want 2", c.cfg.BatchSize)
+ }
+ if c.cfg.MaxPages != 1000 {
+ t.Fatalf("max_pages=%d want 1000", c.cfg.MaxPages)
+ }
+ if c.cfg.RequestDelay != 0.5 {
+ t.Fatalf("request_delay=%v want 0.5", c.cfg.RequestDelay)
+ }
+ if len(c.cfg.ContentFields) != 2 || c.cfg.ContentFields[0] != "title" || c.cfg.ContentFields[1] != "body" {
+ t.Fatalf("content_fields=%v", c.cfg.ContentFields)
+ }
+ if c.cfg.AuthType != "none" || c.cfg.PaginationType != "none" {
+ t.Fatalf("auth=%q pagination=%q", c.cfg.AuthType, c.cfg.PaginationType)
+ }
+}
+
+func TestNewRestAPIConnectorClampsNonPositiveBatchSize(t *testing.T) {
+ withRestAPITestHooks(t)
+ for _, batchSize := range []any{0, -1} {
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": "https://example.com/api",
+ "content_fields": "title",
+ "batch_size": batchSize,
+ })
+ if c.cfg.BatchSize != restAPIDefaultBatchSize {
+ t.Fatalf("batch_size=%v parsed to %d, want %d", batchSize, c.cfg.BatchSize, restAPIDefaultBatchSize)
+ }
+ }
+}
+
+type restAPITestReadCloser struct {
+ reader io.Reader
+ closeErr error
+}
+
+func (b *restAPITestReadCloser) Read(p []byte) (int, error) { return b.reader.Read(p) }
+func (b *restAPITestReadCloser) Close() error { return b.closeErr }
+
+func TestRestAPICloseIdleBodyPreservesBodyAndCloseError(t *testing.T) {
+ closeErr := errors.New("close boom")
+ body := &restAPITestReadCloser{reader: strings.NewReader("hello"), closeErr: closeErr}
+ wrapped := &restAPICloseIdleBody{body: body, transport: &http.Transport{}}
+ got, err := io.ReadAll(wrapped)
+ if err != nil || string(got) != "hello" {
+ t.Fatalf("read data=%q err=%v", got, err)
+ }
+ if err := wrapped.Close(); !errors.Is(err, closeErr) {
+ t.Fatalf("close err=%v want %v", err, closeErr)
+ }
+}
+
+func TestNewRestAPIConnectorValidationErrors(t *testing.T) {
+ withRestAPITestHooks(t)
+ tests := []struct {
+ name string
+ config map[string]any
+ want string
+ }{
+ {name: "missing url", config: map[string]any{"content_fields": "title"}, want: "Invalid REST API config: url"},
+ {name: "unsupported method", config: map[string]any{"url": "https://example.com", "method": "DELETE", "content_fields": "title"}, want: "Unsupported HTTP method 'DELETE'."},
+ {name: "unsupported auth", config: map[string]any{"url": "https://example.com", "auth_type": "jwt", "content_fields": "title"}, want: "Unsupported auth_type 'jwt'."},
+ {name: "unsupported pagination", config: map[string]any{"url": "https://example.com", "pagination_type": "bad", "content_fields": "title"}, want: "Unsupported pagination_type 'bad'."},
+ {name: "missing content fields", config: map[string]any{"url": "https://example.com"}, want: "At least one content field must be configured (content_fields)."},
+ {name: "zero max_pages", config: map[string]any{"url": "https://example.com", "max_pages": 0, "content_fields": "title"}, want: "max_pages must be a positive integer"},
+ {name: "negative max_pages", config: map[string]any{"url": "https://example.com", "max_pages": -1, "content_fields": "title"}, want: "max_pages must be a positive integer"},
+ {name: "bad scheme", config: map[string]any{"url": "ftp://example.com/x", "content_fields": "title"}, want: "Unsupported URL scheme"},
+ {name: "localhost", config: map[string]any{"url": "http://localhost/x", "content_fields": "title"}, want: "localhost is blocked"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ _, err := NewRestAPIConnector(tt.config)
+ if err == nil || !strings.Contains(err.Error(), tt.want) {
+ t.Fatalf("err=%v want contains %q", err, tt.want)
+ }
+ var valErr *ConnectorValidationError
+ if !errors.As(err, &valErr) {
+ t.Fatalf("err=%T want *ConnectorValidationError", err)
+ }
+ })
+ }
+}
+
+func TestNewRestAPIConnectorBlocksPrivateAddress(t *testing.T) {
+ restAPISSRFAllowLoopback = false
+ defer func() { restAPISSRFAllowLoopback = false }()
+ _, err := NewRestAPIConnector(map[string]any{
+ "url": "http://127.0.0.1:8080/api",
+ "content_fields": "title",
+ })
+ if err == nil || !strings.Contains(err.Error(), "resolves to disallowed address") {
+ t.Fatalf("err=%v want private address rejection", err)
+ }
+}
+
+func TestRestAPITextToDict(t *testing.T) {
+ dict := restAPITextToDict(map[string]any{"a": 1, "b": "x"})
+ if dict["a"] != "1" || dict["b"] != "x" {
+ t.Fatalf("dict=%v", dict)
+ }
+ jsonDict := restAPITextToDict(`{"a": "1", "b": "x"}`)
+ if jsonDict["a"] != "1" || jsonDict["b"] != "x" {
+ t.Fatalf("jsonDict=%v", jsonDict)
+ }
+ lines := restAPITextToDict("# comment\na=1\nb = x y\n")
+ if lines["a"] != "1" || lines["b"] != "x y" {
+ t.Fatalf("lines=%v", lines)
+ }
+}
+
+func TestRestAPIAuthPreparation(t *testing.T) {
+ withRestAPITestHooks(t)
+
+ apiKey := mustRestAPIConnector(t, map[string]any{
+ "url": "https://example.com",
+ "content_fields": "title",
+ "auth_type": "api_key_header",
+ "auth_config": map[string]any{"header_name": "X-Key"},
+ "credentials": map[string]any{"api_key": "secret"},
+ })
+ if err := apiKey.prepare(); err != nil {
+ t.Fatalf("prepare api_key: %v", err)
+ }
+ if got := apiKey.authHeaders["X-Key"]; got != "secret" {
+ t.Fatalf("X-Key=%q", got)
+ }
+
+ bearer := mustRestAPIConnector(t, map[string]any{
+ "url": "https://example.com",
+ "content_fields": "title",
+ "auth_type": "bearer",
+ "credentials": map[string]any{"token": "tok"},
+ })
+ if err := bearer.prepare(); err != nil {
+ t.Fatalf("prepare bearer: %v", err)
+ }
+ if got := bearer.authHeaders["Authorization"]; got != "Bearer tok" {
+ t.Fatalf("Authorization=%q", got)
+ }
+
+ basic := mustRestAPIConnector(t, map[string]any{
+ "url": "https://example.com",
+ "content_fields": "title",
+ "auth_type": "basic",
+ "credentials": map[string]any{"username": "u", "password": "p"},
+ })
+ if err := basic.prepare(); err != nil {
+ t.Fatalf("prepare basic: %v", err)
+ }
+ if basic.basicAuth == nil || basic.basicAuth.username != "u" || basic.basicAuth.password != "p" {
+ t.Fatalf("basicAuth=%+v", basic.basicAuth)
+ }
+
+ missing := mustRestAPIConnector(t, map[string]any{
+ "url": "https://example.com",
+ "content_fields": "title",
+ "auth_type": "bearer",
+ })
+ err := missing.prepare()
+ var credErr *ConnectorMissingCredentialError
+ if !errors.As(err, &credErr) || credErr.Message != "REST API (bearer) requires 'token' in credentials" {
+ t.Fatalf("err=%v want missing credential error", err)
+ }
+}
+
+func TestRestAPIFieldExtraction(t *testing.T) {
+ withRestAPITestHooks(t)
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": "https://example.com",
+ "content_fields": "title",
+ "field_type_hints": map[string]any{"meta.count": "number", "meta.name": "string"},
+ "field_default_values": map[string]any{"missing": "fallback"},
+ })
+ item := map[string]any{
+ "title": "T",
+ "meta": map[string]any{"count": json.Number("2"), "name": 7},
+ "tags": []any{"a", "b"},
+ }
+ values := extractRestAPIFieldValues(item, "meta.count")
+ if len(values) != 1 || values[0] != json.Number("2") {
+ t.Fatalf("meta.count=%v", values)
+ }
+ if got := c.getTypedFieldValue("meta.count", item); got != int64(2) {
+ t.Fatalf("typed meta.count=%v (%T)", got, got)
+ }
+ if got := c.getTypedFieldValue("meta.name", item); got != "7" {
+ t.Fatalf("typed meta.name=%v", got)
+ }
+ tags := extractRestAPIFieldValues(item, "tags[*]")
+ if len(tags) != 2 || tags[0] != "a" || tags[1] != "b" {
+ t.Fatalf("tags[*]=%v", tags)
+ }
+ if got := c.getTypedFieldValue("missing", item); got != "fallback" {
+ t.Fatalf("default=%v", got)
+ }
+}
+
+func TestRestAPIParseDatetime(t *testing.T) {
+ tests := []struct {
+ input any
+ want string
+ }{
+ {input: "2026-08-14T10:00:00Z", want: "2026-08-14T10:00:00Z"},
+ {input: "2026-08-14 10:00:00", want: "2026-08-14T10:00:00Z"},
+ {input: "2026-08-14T18:00:00+08:00", want: "2026-08-14T10:00:00Z"},
+ {input: "2026-08-14", want: "2026-08-14T00:00:00Z"},
+ {input: time.Date(2026, 8, 14, 10, 0, 0, 0, time.UTC).Unix(), want: "2026-08-14T10:00:00Z"},
+ }
+ for _, tt := range tests {
+ got := parseRestAPIDatetime(tt.input)
+ if got == nil {
+ t.Fatalf("parse(%v) = nil", tt.input)
+ }
+ if got.UTC().Format(time.RFC3339) != tt.want {
+ t.Fatalf("parse(%v)=%s want %s", tt.input, got.UTC().Format(time.RFC3339), tt.want)
+ }
+ }
+}
+
+func TestRestAPIItemToDocument(t *testing.T) {
+ withRestAPITestHooks(t)
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": "https://example.com",
+ "id_field": "id",
+ "content_fields": "title",
+ "metadata_fields": "meta.count",
+ "poll_timestamp_field": "updated",
+ })
+ item := map[string]any{
+ "id": "abc",
+ "title": "Hello <b>World</b>",
+ "meta": map[string]any{"count": json.Number("3")},
+ "updated": "2026-08-14T10:00:00Z",
+ }
+ doc, err := c.itemToDocument(item)
+ if err != nil {
+ t.Fatalf("itemToDocument: %v", err)
+ }
+ wantID := restAPIHash128("rest_api:" + "abc")
+ if doc.SourceID != wantID {
+ t.Fatalf("SourceID=%s want %s", doc.SourceID, wantID)
+ }
+ if doc.SemanticIdentifier != "Hello World" {
+ t.Fatalf("sem=%q", doc.SemanticIdentifier)
+ }
+ if string(doc.Blob) != "Hello World" {
+ t.Fatalf("blob=%q", doc.Blob)
+ }
+ if doc.Extension != ".txt" {
+ t.Fatalf("extension=%q", doc.Extension)
+ }
+ if doc.Metadata["meta.count"] != int64(3) {
+ t.Fatalf("metadata=%v", doc.Metadata)
+ }
+ wantTime := time.Date(2026, 8, 14, 10, 0, 0, 0, time.UTC)
+ if !doc.UpdatedAt.Equal(wantTime) {
+ t.Fatalf("UpdatedAt=%v want %v", doc.UpdatedAt, wantTime)
+ }
+ if doc.Fingerprint != contentFingerprint(doc.Blob) {
+ t.Fatalf("fingerprint mismatch")
+ }
+}
+
+func TestRestAPIHash128AndStableText(t *testing.T) {
+ if got := restAPIStableText(map[string]any{"a": "x", "b": 1}); got != "{'a': 'x', 'b': 1}" {
+ t.Fatalf("stable text dict=%q", got)
+ }
+ if got := restAPIStableText([]any{"a", 1, true, nil}); got != "['a', 1, True, None]" {
+ t.Fatalf("stable text list=%q", got)
+ }
+ if got := restAPIQuoteString("a'b\n"); got != `'a\'b\n'` {
+ t.Fatalf("quoted string=%q", got)
+ }
+ if got := restAPIValueText(json.Number("1.0")); got != "1.0" {
+ t.Fatalf("restAPIValueText float=%q", got)
+ }
+ if got := restAPIValueText(json.Number("3")); got != "3" {
+ t.Fatalf("restAPIValueText int=%q", got)
+ }
+ if got := restAPIHash128("rest_api:abc"); len(got) != 32 {
+ t.Fatalf("hash length=%d want 32", len(got))
+ }
+}
+
+func TestRestAPIJSONPathAndExtractItems(t *testing.T) {
+ response := map[string]any{
+ "data": map[string]any{
+ "items": []any{
+ map[string]any{"id": "1"},
+ map[string]any{"id": "2"},
+ },
+ },
+ "paging": map[string]any{"next": "cursor-1"},
+ }
+ items := restAPIExtractItems(response, "$.data.items")
+ if len(items) != 2 {
+ t.Fatalf("items=%d want 2", len(items))
+ }
+ if got := restAPIExtractNextCursor(response, map[string]any{"next_cursor_field": "paging.next"}); got != "" {
+ t.Fatalf("next_cursor_field=%q want empty (field is top-level only)", got)
+ }
+ if got := restAPIExtractNextCursor(response, map[string]any{"next_cursor_path": "$.paging.next"}); got != "cursor-1" {
+ t.Fatalf("next_cursor_path=%q", got)
+ }
+ fallback := restAPIExtractItems(map[string]any{"results": []any{map[string]any{"id": "9"}}}, "")
+ if len(fallback) != 1 {
+ t.Fatalf("fallback items=%d", len(fallback))
+ }
+}
+
+func TestRestAPIContentTemplate(t *testing.T) {
+ withRestAPITestHooks(t)
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": "https://example.com",
+ "content_fields": "title",
+ "metadata_fields": "meta.count",
+ "content_template": "Title: {title} / Count: {meta_count}",
+ })
+ item := map[string]any{"title": "T", "meta": map[string]any{"count": json.Number("3")}}
+ if got := c.renderContentTemplate(item); got != "Title: T / Count: 3" {
+ t.Fatalf("rendered=%q", got)
+ }
+ broken := mustRestAPIConnector(t, map[string]any{
+ "url": "https://example.com",
+ "content_fields": "title",
+ "content_template": "{",
+ })
+ if got := broken.renderContentTemplate(item); got != "T" {
+ t.Fatalf("fallback rendered=%q", got)
+ }
+}
+
+func TestRestAPIFetchPageIntegration(t *testing.T) {
+ withRestAPITestHooks(t)
+
+ var requests atomic.Int32
+ var mu sync.Mutex
+ var gotPath, gotAuth string
+ var gotMethod string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ mu.Lock()
+ gotPath = r.URL.Path
+ gotAuth = r.Header.Get("Authorization")
+ gotMethod = r.Method
+ mu.Unlock()
+ w.Header().Set("Content-Type", "application/json")
+ w.Write([]byte(`{"items": [{"id": "1"}]}`))
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL + "/items/{tenant}",
+ "query_params": map[string]any{"tenant": "abc"},
+ "content_fields": "title",
+ "auth_type": "bearer",
+ "credentials": map[string]any{"token": "tok"},
+ })
+ if _, err := c.fetchPage(context.Background(), map[string]any{"page": 1}); err != nil {
+ t.Fatalf("fetchPage: %v", err)
+ }
+ mu.Lock()
+ path, auth, method := gotPath, gotAuth, gotMethod
+ mu.Unlock()
+ if path != "/items/abc" {
+ t.Fatalf("path=%q want /items/abc", path)
+ }
+ if auth != "Bearer tok" {
+ t.Fatalf("auth=%q", auth)
+ }
+ if method != "GET" {
+ t.Fatalf("method=%q", method)
+ }
+ if requests.Load() != 1 {
+ t.Fatalf("requests=%d", requests.Load())
+ }
+}
+
+func TestRestAPIFetchPagePOST(t *testing.T) {
+ withRestAPITestHooks(t)
+ var mu sync.Mutex
+ var gotMethod, gotBody, gotContentType string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ gotMethod = r.Method
+ gotContentType = r.Header.Get("Content-Type")
+ body, _ := io.ReadAll(r.Body)
+ gotBody = string(body)
+ mu.Unlock()
+ w.Header().Set("Content-Type", "application/json")
+ w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "method": "POST",
+ "content_fields": "title",
+ "request_body": map[string]any{"q": "x"},
+ })
+ if _, err := c.fetchPage(context.Background(), nil); err != nil {
+ t.Fatalf("fetchPage: %v", err)
+ }
+ mu.Lock()
+ method, contentType, body := gotMethod, gotContentType, gotBody
+ mu.Unlock()
+ if method != "POST" {
+ t.Fatalf("method=%q", method)
+ }
+ if contentType != "application/json" {
+ t.Fatalf("content-type=%q", contentType)
+ }
+ if !strings.Contains(body, `"q":"x"`) {
+ t.Fatalf("body=%q", body)
+ }
+}
+
+func TestRestAPIFetchPageErrorMapping(t *testing.T) {
+ withRestAPITestHooks(t)
+ tests := []struct {
+ name string
+ status int
+ body string
+ wantType any
+ wantSubstr string
+ }{
+ {name: "unauthorized", status: 401, body: `{}`, wantType: &ConnectorMissingCredentialError{}, wantSubstr: "REST API authentication failed with status 401"},
+ {name: "bad request", status: 400, body: `{}`, wantType: &ConnectorValidationError{}, wantSubstr: "REST API request failed with non-retriable client error status 400"},
+ {name: "non json", status: 200, body: `not-json`, wantType: &ConnectorValidationError{}, wantSubstr: "REST API response is not valid JSON"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(tt.status)
+ w.Write([]byte(tt.body))
+ }))
+ defer server.Close()
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ })
+ _, err := c.fetchPage(context.Background(), nil)
+ if err == nil || !strings.Contains(err.Error(), tt.wantSubstr) {
+ t.Fatalf("err=%v want contains %q", err, tt.wantSubstr)
+ }
+ switch tt.wantType.(type) {
+ case *ConnectorMissingCredentialError:
+ var want *ConnectorMissingCredentialError
+ if !errors.As(err, &want) {
+ t.Fatalf("err=%T want missing credential error", err)
+ }
+ case *ConnectorValidationError:
+ var want *ConnectorValidationError
+ if !errors.As(err, &want) {
+ t.Fatalf("err=%T want validation error", err)
+ }
+ }
+ })
+ }
+}
+
+func TestRestAPIFetchPageRetries(t *testing.T) {
+ withRestAPITestHooks(t)
+ var attempts atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ n := attempts.Add(1)
+ if n < 3 {
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.Write([]byte(`{"ok": true}`))
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ })
+ if _, err := c.fetchPage(context.Background(), nil); err != nil {
+ t.Fatalf("fetchPage: %v", err)
+ }
+ if attempts.Load() != 3 {
+ t.Fatalf("attempts=%d want 3", attempts.Load())
+ }
+}
+
+func TestRestAPIFetchPage429RetryAfter(t *testing.T) {
+ withRestAPITestHooks(t)
+ var attempts atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if attempts.Add(1) == 1 {
+ w.Header().Set("Retry-After", "0")
+ w.WriteHeader(http.StatusTooManyRequests)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.Write([]byte(`{"ok": true}`))
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ })
+ if _, err := c.fetchPage(context.Background(), nil); err != nil {
+ t.Fatalf("fetchPage: %v", err)
+ }
+ if attempts.Load() != 2 {
+ t.Fatalf("attempts=%d want 2", attempts.Load())
+ }
+}
+
+func TestRestAPIFetchPageRedirectStripsAuth(t *testing.T) {
+ withRestAPITestHooks(t)
+ var mu sync.Mutex
+ var gotAuth, gotMethod string
+ final := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ gotAuth = r.Header.Get("Authorization")
+ gotMethod = r.Method
+ mu.Unlock()
+ w.Header().Set("Content-Type", "application/json")
+ w.Write([]byte(`{}`))
+ }))
+ defer final.Close()
+ redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Redirect(w, r, final.URL, http.StatusFound)
+ }))
+ defer redirector.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": redirector.URL,
+ "method": "POST",
+ "content_fields": "title",
+ "auth_type": "bearer",
+ "credentials": map[string]any{"token": "tok"},
+ })
+ if _, err := c.fetchPage(context.Background(), nil); err != nil {
+ t.Fatalf("fetchPage: %v", err)
+ }
+ mu.Lock()
+ auth, method := gotAuth, gotMethod
+ mu.Unlock()
+ if auth != "" {
+ t.Fatalf("cross-origin auth=%q want stripped", auth)
+ }
+ if method != "GET" {
+ t.Fatalf("redirect method=%q want GET", method)
+ }
+}
+
+func TestRestAPISyncSessionPagination(t *testing.T) {
+ withRestAPITestHooks(t)
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ page := r.URL.Query().Get("page")
+ items := []any{}
+ switch page {
+ case "", "1":
+ items = []any{
+ map[string]any{"id": "1", "title": "One", "updated": "2026-08-14T10:00:00Z"},
+ map[string]any{"id": "2", "title": "Two", "updated": "2026-08-14T11:00:00Z"},
+ }
+ case "2":
+ items = []any{
+ map[string]any{"id": "3", "title": "Three", "updated": "2026-08-14T12:00:00Z"},
+ map[string]any{"id": "4", "title": "Four", "updated": "2026-08-14T13:00:00Z"},
+ }
+ default:
+ items = []any{map[string]any{"id": "5", "title": "Five", "updated": "2026-08-14T14:00:00Z"}}
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(map[string]any{"items": items})
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ "id_field": "id",
+ "pagination_type": "page",
+ "pagination_config": map[string]any{"page_size": 2},
+ "batch_size": 2,
+ "request_delay": 0,
+ })
+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+
+ var total int
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) > 2 {
+ t.Fatalf("batch size=%d", len(batch.Documents))
+ }
+ total += len(batch.Documents)
+ }
+ if total != 5 {
+ t.Fatalf("total=%d want 5", total)
+ }
+ if requests.Load() != 3 {
+ t.Fatalf("requests=%d want 3", requests.Load())
+ }
+}
+
+func TestRestAPISyncSessionWindowFilter(t *testing.T) {
+ withRestAPITestHooks(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(map[string]any{
+ "items": []any{
+ map[string]any{"id": "1", "title": "One", "updated": "2026-08-14T10:00:00Z"},
+ map[string]any{"id": "2", "title": "Two", "updated": "2026-08-14T12:00:00Z"},
+ },
+ })
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ "id_field": "id",
+ "poll_timestamp_field": "updated",
+ })
+ start := time.Date(2026, 8, 14, 11, 0, 0, 0, time.UTC)
+ end := time.Date(2026, 8, 14, 13, 0, 0, 0, time.UTC)
+ session, err := c.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != restAPIHash128("rest_api:2") {
+ t.Fatalf("documents=%d want window-filtered id 2", len(batch.Documents))
+ }
+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("err=%v want EOF", err)
+ }
+}
+
+func TestRestAPIValidateLive(t *testing.T) {
+ withRestAPITestHooks(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.Write([]byte(`{"items": []}`))
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ })
+ if err := c.ValidateLive(context.Background()); err != nil {
+ t.Fatalf("ValidateLive: %v", err)
+ }
+ if err := c.Validate(context.Background()); err != nil {
+ t.Fatalf("Validate: %v", err)
+ }
+}
+
+func TestRestAPIMaxPages(t *testing.T) {
+ withRestAPITestHooks(t)
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(map[string]any{
+ "items": []any{map[string]any{"id": "1", "title": "One"}, map[string]any{"id": "2", "title": "Two"}},
+ })
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ "id_field": "id",
+ "pagination_type": "page",
+ "pagination_config": map[string]any{"page_size": 2},
+ "max_pages": 1,
+ "request_delay": 0,
+ })
+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("documents=%d want 2", len(batch.Documents))
+ }
+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("err=%v want EOF", err)
+ }
+ if requests.Load() != 1 {
+ t.Fatalf("requests=%d want 1", requests.Load())
+ }
+}
+
+func TestRestAPIOpenPruneUnsupported(t *testing.T) {
+ withRestAPITestHooks(t)
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": "https://example.com",
+ "content_fields": "title",
+ })
+ _, err := c.OpenPrune(context.Background(), PruneRequest{})
+ if !errors.Is(err, ErrPruneUnsupported) {
+ t.Fatalf("err=%v want ErrPruneUnsupported", err)
+ }
+}
+
+func restAPICheckpointCursor(t *testing.T, batch SyncBatch) restAPISyncCursor {
+ t.Helper()
+ if batch.Checkpoint == nil || batch.Checkpoint.Cursor == "" {
+ t.Fatalf("checkpoint is missing")
+ }
+ var cursor restAPISyncCursor
+ if err := json.Unmarshal([]byte(batch.Checkpoint.Cursor), &cursor); err != nil {
+ t.Fatalf("decode checkpoint: %v", err)
+ }
+ return cursor
+}
+
+func TestRestAPIFetchPageServerErrorIsTransient(t *testing.T) {
+ withRestAPITestHooks(t)
+ var attempts atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ attempts.Add(1)
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ })
+ _, err := c.fetchPage(context.Background(), nil)
+ if err == nil || !strings.Contains(err.Error(), "http 500") {
+ t.Fatalf("err=%v want message containing http 500", err)
+ }
+ if int(attempts.Load()) != restAPIRetryTries {
+ t.Fatalf("attempts=%d want %d", attempts.Load(), restAPIRetryTries)
+ }
+}
+
+func TestRestAPIFetchPage429ExhaustionIsTransient(t *testing.T) {
+ withRestAPITestHooks(t)
+ var attempts atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ attempts.Add(1)
+ w.WriteHeader(http.StatusTooManyRequests)
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ })
+ _, err := c.fetchPage(context.Background(), nil)
+ if err == nil || !strings.Contains(err.Error(), "too many requests") {
+ t.Fatalf("err=%v want message containing too many requests", err)
+ }
+ var rateErr *RateLimitTriedTooManyTimesError
+ if !errors.As(err, &rateErr) {
+ t.Fatalf("err=%T want RateLimitTriedTooManyTimesError", err)
+ }
+ if int(attempts.Load()) != restAPI429MaxWaits {
+ t.Fatalf("attempts=%d want %d", attempts.Load(), restAPI429MaxWaits)
+ }
+}
+
+func TestRestAPISyncSessionNoneNoCheckpoint(t *testing.T) {
+ withRestAPITestHooks(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode([]any{
+ map[string]any{"id": "1", "title": "One"},
+ map[string]any{"id": "2", "title": "Two"},
+ map[string]any{"id": "3", "title": "Three"},
+ })
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ "id_field": "id",
+ "batch_size": 2,
+ })
+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if batch.Checkpoint != nil {
+ t.Fatalf("pagination none checkpoint=%+v want nil", batch.Checkpoint)
+ }
+ }
+}
+
+func TestRestAPISyncSessionPageResume(t *testing.T) {
+ withRestAPITestHooks(t)
+ var mu sync.Mutex
+ var requested []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ page := r.URL.Query().Get("page")
+ mu.Lock()
+ requested = append(requested, page)
+ mu.Unlock()
+ items := []any{}
+ switch page {
+ case "1":
+ items = []any{
+ map[string]any{"id": "1", "title": "One"},
+ map[string]any{"id": "2", "title": "Two"},
+ }
+ case "2":
+ items = []any{
+ map[string]any{"id": "3", "title": "Three"},
+ map[string]any{"id": "4", "title": "Four"},
+ }
+ default:
+ items = []any{map[string]any{"id": "5", "title": "Five"}}
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(map[string]any{"items": items})
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ "id_field": "id",
+ "pagination_type": "page",
+ "pagination_config": map[string]any{"page_size": 2},
+ "batch_size": 2,
+ "request_delay": 0,
+ })
+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ session.Close()
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 ||
+ batch.Documents[0].SourceID != restAPIHash128("rest_api:1") ||
+ batch.Documents[1].SourceID != restAPIHash128("rest_api:2") {
+ t.Fatalf("documents=%v want ids 1,2", batch.Documents)
+ }
+ cursor := restAPICheckpointCursor(t, batch)
+ if cursor.Page != 2 {
+ t.Fatalf("cursor=%+v want page 2", cursor)
+ }
+
+ resumed, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: batch.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync: %v", err)
+ }
+ defer resumed.Close()
+ var sourceIDs []string
+ for {
+ b, err := resumed.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("resumed NextBatch: %v", err)
+ }
+ for _, doc := range b.Documents {
+ sourceIDs = append(sourceIDs, doc.SourceID)
+ }
+ }
+ want := []string{restAPIHash128("rest_api:3"), restAPIHash128("rest_api:4"), restAPIHash128("rest_api:5")}
+ if strings.Join(sourceIDs, ",") != strings.Join(want, ",") {
+ t.Fatalf("sourceIDs=%v want %v", sourceIDs, want)
+ }
+ mu.Lock()
+ got := append([]string(nil), requested...)
+ mu.Unlock()
+ if len(got) != 3 || got[0] != "1" || got[1] != "2" || got[2] != "3" {
+ t.Fatalf("requested pages=%v want [1 2 3]", got)
+ }
+}
+
+func TestRestAPISyncSessionOffsetResume(t *testing.T) {
+ withRestAPITestHooks(t)
+ var mu sync.Mutex
+ var requested []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ offset := r.URL.Query().Get("offset")
+ mu.Lock()
+ requested = append(requested, offset)
+ mu.Unlock()
+ items := []any{}
+ switch offset {
+ case "0":
+ items = []any{
+ map[string]any{"id": "1", "title": "One"},
+ map[string]any{"id": "2", "title": "Two"},
+ }
+ case "2":
+ items = []any{
+ map[string]any{"id": "3", "title": "Three"},
+ map[string]any{"id": "4", "title": "Four"},
+ }
+ default:
+ items = []any{map[string]any{"id": "5", "title": "Five"}}
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(map[string]any{"items": items})
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ "id_field": "id",
+ "pagination_type": "offset",
+ "pagination_config": map[string]any{"limit": 2},
+ "batch_size": 2,
+ "request_delay": 0,
+ })
+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ session.Close()
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 ||
+ batch.Documents[0].SourceID != restAPIHash128("rest_api:1") ||
+ batch.Documents[1].SourceID != restAPIHash128("rest_api:2") {
+ t.Fatalf("documents=%v want ids 1,2", batch.Documents)
+ }
+ cursor := restAPICheckpointCursor(t, batch)
+ if cursor.Offset != 2 {
+ t.Fatalf("cursor=%+v want offset 2", cursor)
+ }
+
+ resumed, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: batch.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync: %v", err)
+ }
+ defer resumed.Close()
+ var sourceIDs []string
+ for {
+ b, err := resumed.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("resumed NextBatch: %v", err)
+ }
+ for _, doc := range b.Documents {
+ sourceIDs = append(sourceIDs, doc.SourceID)
+ }
+ }
+ want := []string{restAPIHash128("rest_api:3"), restAPIHash128("rest_api:4"), restAPIHash128("rest_api:5")}
+ if strings.Join(sourceIDs, ",") != strings.Join(want, ",") {
+ t.Fatalf("sourceIDs=%v want %v", sourceIDs, want)
+ }
+ mu.Lock()
+ got := append([]string(nil), requested...)
+ mu.Unlock()
+ if len(got) != 3 || got[0] != "0" || got[1] != "2" || got[2] != "4" {
+ t.Fatalf("requested offsets=%v want [0 2 4]", got)
+ }
+}
+
+func TestRestAPISyncSessionCursorResume(t *testing.T) {
+ withRestAPITestHooks(t)
+ var mu sync.Mutex
+ var requested []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ cursor := r.URL.Query().Get("cursor")
+ mu.Lock()
+ requested = append(requested, cursor)
+ mu.Unlock()
+ resp := map[string]any{"items": []any{}}
+ switch cursor {
+ case "":
+ resp["items"] = []any{
+ map[string]any{"id": "1", "title": "One"},
+ map[string]any{"id": "2", "title": "Two"},
+ }
+ resp["next_page_token"] = "t2"
+ case "t2":
+ resp["items"] = []any{
+ map[string]any{"id": "3", "title": "Three"},
+ map[string]any{"id": "4", "title": "Four"},
+ }
+ resp["next_page_token"] = "t3"
+ default:
+ resp["items"] = []any{map[string]any{"id": "5", "title": "Five"}}
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(resp)
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ "id_field": "id",
+ "pagination_type": "cursor",
+ "pagination_config": map[string]any{"next_cursor_field": "next_page_token"},
+ "batch_size": 2,
+ "request_delay": 0,
+ })
+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ session.Close()
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 ||
+ batch.Documents[0].SourceID != restAPIHash128("rest_api:1") ||
+ batch.Documents[1].SourceID != restAPIHash128("rest_api:2") {
+ t.Fatalf("documents=%v want ids 1,2", batch.Documents)
+ }
+ cursor := restAPICheckpointCursor(t, batch)
+ if cursor.Cursor != "t2" {
+ t.Fatalf("cursor=%+v want cursor t2", cursor)
+ }
+
+ resumed, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: batch.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync: %v", err)
+ }
+ defer resumed.Close()
+ var sourceIDs []string
+ for {
+ b, err := resumed.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("resumed NextBatch: %v", err)
+ }
+ for _, doc := range b.Documents {
+ sourceIDs = append(sourceIDs, doc.SourceID)
+ }
+ }
+ want := []string{restAPIHash128("rest_api:3"), restAPIHash128("rest_api:4"), restAPIHash128("rest_api:5")}
+ if strings.Join(sourceIDs, ",") != strings.Join(want, ",") {
+ t.Fatalf("sourceIDs=%v want %v", sourceIDs, want)
+ }
+ mu.Lock()
+ got := append([]string(nil), requested...)
+ mu.Unlock()
+ if len(got) != 3 || got[0] != "" || got[1] != "t2" || got[2] != "t3" {
+ t.Fatalf("requested cursors=%v want [\"\" t2 t3]", got)
+ }
+}
+
+func TestRestAPISyncSessionCheckpointWaitsForPageBoundary(t *testing.T) {
+ withRestAPITestHooks(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ page := r.URL.Query().Get("page")
+ items := []any{}
+ switch page {
+ case "1":
+ items = []any{
+ map[string]any{"id": "1", "title": "One"},
+ map[string]any{"id": "2", "title": "Two"},
+ map[string]any{"id": "3", "title": "Three"},
+ map[string]any{"id": "4", "title": "Four"},
+ }
+ default:
+ items = []any{map[string]any{"id": "5", "title": "Five"}}
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(map[string]any{"items": items})
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ "id_field": "id",
+ "pagination_type": "page",
+ "pagination_config": map[string]any{"page_size": 4},
+ "batch_size": 2,
+ "request_delay": 0,
+ })
+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(first.Documents) != 2 {
+ t.Fatalf("documents=%d want 2", len(first.Documents))
+ }
+ if first.Checkpoint != nil {
+ t.Fatalf("mid-page checkpoint=%+v want nil", first.Checkpoint)
+ }
+ second, err := session.NextBatch(context.Background())
+ session.Close()
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(second.Documents) != 2 {
+ t.Fatalf("documents=%d want 2", len(second.Documents))
+ }
+ cursor := restAPICheckpointCursor(t, second)
+ if cursor.Page != 2 {
+ t.Fatalf("cursor=%+v want page 2", cursor)
+ }
+
+ resumed, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: second.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync: %v", err)
+ }
+ defer resumed.Close()
+ b, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resumed NextBatch: %v", err)
+ }
+ if len(b.Documents) != 1 || b.Documents[0].SourceID != restAPIHash128("rest_api:5") {
+ t.Fatalf("resumed documents=%v want id 5", b.Documents)
+ }
+ if _, err := resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("err=%v want EOF", err)
+ }
+}
diff --git a/internal/syncer/prune_runner.go b/internal/syncer/prune_runner.go
index 8da242c6d3..3cbbeb4db8 100644
--- a/internal/syncer/prune_runner.go
+++ b/internal/syncer/prune_runner.go
@@ -20,8 +20,11 @@ import (
"context"
"errors"
"io"
+ "ragflow/internal/common"
"ragflow/internal/service"
syncerconnector "ragflow/internal/syncer/connector"
+
+ "go.uber.org/zap"
)
// PruneRunner executes one PRUNE task after collecting a full slim snapshot.
@@ -47,6 +50,15 @@ func (r *PruneRunner) Run(ctx context.Context, taskContext service.SyncTaskConte
KBID: taskContext.Knowledgebase.ID,
})
if err != nil {
+ if errors.Is(err, syncerconnector.ErrPruneUnsupported) {
+ // Connectors without a slim snapshot interface (e.g. REST API)
+ // complete PRUNE as a no-op without deleting anything.
+ if err := r.checkCanceled(ctx, taskContext.Task.ID); err != nil {
+ return "", err
+ }
+ common.Warn("prune unsupported by connector, completing as no-op", zap.String("task_id", taskContext.Task.ID), zap.Error(err))
+ return r.taskService.CompletePrune(ctx, taskContext, 0)
+ }
return "", err
}
defer session.Close()
diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go
index 4a7be97404..856a88ff87 100644
--- a/internal/syncer/syncer.go
+++ b/internal/syncer/syncer.go
@@ -175,6 +175,7 @@ func registerBuiltInConnectors(registry *syncerconnector.Registry) {
registerDAOConnector(registry, "gmail", syncerconnector.NewGmailConnector)
registerDAOConnector(registry, "google-drive", syncerconnector.NewGoogleDriveConnector)
registerDAOConnector(registry, "google_drive", syncerconnector.NewGoogleDriveConnector)
+ registerDAOConnector(registry, "rest_api", syncerconnector.NewRestAPIConnector)
registerDAOConnector(registry, "mysql", syncerconnector.NewMySQLConnector)
registerDAOConnector(registry, "postgresql", syncerconnector.NewPostgreSQLConnector)
}