RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(syncer): Add REST API connector (#18256)
mkaaadc3b977f7b6a8
diff --git a/internal/handler/connector.go b/internal/handler/connector.goindex 796f2ae9c0..a8ab41c8e5 100644--- a/internal/handler/connector.go+++ b/internal/handler/connector.go@@ -29,6 +29,7 @@ import ("ragflow/internal/common""ragflow/internal/entity""ragflow/internal/service"+ syncerconnector "ragflow/internal/syncer/connector")type connectorServiceIface interface {@@ -361,8 +362,17 @@ func (h *ConnectorHandler) TestConnector(c *gin.Context) {return}if err != nil && !errors.Is(err, service.ErrConnectorNoAuth) && !errors.Is(err, service.ErrConnectorNotFound) {- // Validation failure (e.g. missing credentials): mirror Python's DATA_ERROR with data=false.- common.ResponseWithCodeData(c, common.CodeDataError, false, err.Error())+ // Schema/credential validation failures map to DATA_ERROR;+ // anything unexpected falls back to SERVER_ERROR.+ var (+ valErr *syncerconnector.ConnectorValidationError+ credErr *syncerconnector.ConnectorMissingCredentialError+ )+ if errors.As(err, &valErr) || errors.As(err, &credErr) {+ common.ResponseWithCodeData(c, common.CodeDataError, false, err.Error())+ return+ }+ common.ResponseWithCodeData(c, common.CodeServerError, false, "REST API connector validation failed, please check logs.")return}if connectorErrorResponse(c, err) {diff --git a/internal/handler/connector_test.go b/internal/handler/connector_test.goindex 3d420bf5a1..3b2d744a44 100644--- a/internal/handler/connector_test.go+++ b/internal/handler/connector_test.go@@ -15,6 +15,7 @@ import ("ragflow/internal/common""ragflow/internal/entity""ragflow/internal/service"+ syncerconnector "ragflow/internal/syncer/connector")type fakeConnectorService struct {@@ -265,10 +266,20 @@ func TestConnectorHandlerTestConnector(t *testing.T) {wantCode: common.CodeArgumentError,},{- name: "validation failure",- err: fmt.Errorf("connector credentials are missing"),+ name: "schema validation failure",+ err: &syncerconnector.ConnectorValidationError{Message: "At least one content field must be configured (content_fields)."},wantCode: common.CodeDataError,},+ {+ name: "missing credential failure",+ err: &syncerconnector.ConnectorMissingCredentialError{Message: "REST API (bearer) requires 'token' in credentials"},+ wantCode: common.CodeDataError,+ },+ {+ name: "unexpected failure",+ err: fmt.Errorf("boom"),+ wantCode: common.CodeServerError,+ },}for _, tt := range tests {diff --git a/internal/syncer/connector/rest_api.go b/internal/syncer/connector/rest_api.gonew file mode 100644index 0000000000..9d08756340--- /dev/null+++ b/internal/syncer/connector/rest_api.go@@ -0,0 +1,2211 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "crypto/tls"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "math"+ "math/rand"+ "net"+ "net/http"+ "net/url"+ "regexp"+ "sort"+ "strconv"+ "strings"+ "time"+ "unicode"+)++// RestAPIConnector is a configuration-driven REST API data source connector.+//+// It mirrors common/data_source/rest_api_connector.py: the same config schema,+// auth derivation, SSRF protection, pagination, JSON extraction and+// item-to-document mapping. The connector is read-only over HTTP(S).+type RestAPIConnector struct {+ cfg *restAPIConfig+ baseURL string+ urlParams map[string]string+ credentials map[string]any++ authHeaders map[string]string+ basicAuth *restAPIBasicAuth+ prepared bool+}++// restAPIConfig is the validated connector configuration.+type restAPIConfig struct {+ URL string+ Method string+ Headers map[string]string+ QueryParams map[string]string+ AuthType string+ AuthConfig map[string]any+ ItemsPath string+ IDField string+ ContentFields []string+ MetadataFields []string+ PaginationType string+ PaginationConfig map[string]any+ PollTimestampField string+ RequestBody map[string]any+ FieldTypeHints map[string]string+ FieldDefaultValues map[string]any+ ContentTemplate string+ BatchSize int+ MaxPages int+ RequestDelay float64+}++// restAPIBasicAuth carries HTTP basic credentials.+type restAPIBasicAuth struct {+ username string+ password string+}++const (+ restAPIDefaultBatchSize = 2 // default per-batch document count+ restAPIDefaultMaxPages = 1000 // default page fetch cap+ restAPIDefaultRequestDelay = 0.5+ restAPIRequestTimeout = 60 * time.Second+ restAPIMaxRedirects = 5+)++const (+ restAPIAuthNone = "none"+ restAPIAuthAPIKeyHeader = "api_key_header"+ restAPIAuthBearer = "bearer"+ restAPIAuthBasic = "basic"++ restAPIPaginationNone = "none"+ restAPIPaginationPage = "page"+ restAPIPaginationOffset = "offset"+ restAPIPaginationCursor = "cursor"+)++// Connector error types let callers (e.g. the connector /test endpoint) map+// failures to stable response codes.+type (+ // ConnectorValidationError mirrors ConnectorValidationError.+ ConnectorValidationError struct{ Message string }+ // ConnectorMissingCredentialError mirrors ConnectorMissingCredentialError.+ ConnectorMissingCredentialError struct{ Message string }+ // RateLimitTriedTooManyTimesError mirrors RateLimitTriedTooManyTimesError.+ RateLimitTriedTooManyTimesError struct{ Message string }+)++func (e *ConnectorValidationError) Error() string { return e.Message }+func (e *ConnectorMissingCredentialError) Error() string { return e.Message }+func (e *RateLimitTriedTooManyTimesError) Error() string { return e.Message }++// ErrPruneUnsupported is returned by OpenPrune for connectors that do not+// support deleted-file pruning; prune tasks then complete without deleting+// anything.+var ErrPruneUnsupported = errors.New("rest API connector does not support prune")++// Retry/backoff knobs for page fetches and rate limiting. They are package+// variables so tests can shrink the delays.+var (+ restAPIRetryTries = 5+ restAPIRetryBaseDelay = time.Second+ restAPIRetryMaxDelay = 30 * time.Second+ restAPIRetryBackoff = 2+ restAPIRetryJitter = time.Second+ restAPI429MaxWaits = 30+ restAPI429DefaultWait = 30 * time.Second+)++// restAPISSRFAllowLoopback is a test hook that lets unit tests exercise the+// real HTTP path against httptest servers bound to loopback. Production code+// keeps it false so loopback/private endpoints stay blocked.+var restAPISSRFAllowLoopback bool++// NewRestAPIConnector parses a connector config and returns a connector. It+// performs schema validation and the base-URL SSRF check but performs no+// network I/O. Credentials are read from config["credentials"].+func NewRestAPIConnector(config map[string]any) (*RestAPIConnector, error) {+ cfg, err := parseRestAPIConfig(config)+ if err != nil {+ return nil, err+ }+ if err := validateRestAPIURLForSSRF(cfg.URL); err != nil {+ return nil, err+ }++ parsed, err := url.Parse(cfg.URL)+ if err != nil {+ return nil, &ConnectorValidationError{Message: "Invalid REST API config: url"}+ }++ // Keep the raw path so "{key}" template placeholders survive URL parsing+ // (url.URL.String() would percent-encode them).+ path := parsed.RawPath+ if path == "" {+ path = parsed.Path+ }+ c := &RestAPIConnector{+ cfg: cfg,+ baseURL: parsed.Scheme + "://" + parsed.Host + path,+ urlParams: restAPIURLQueryParams(parsed),+ credentials: restAPICredentialsFromConfig(config),+ authHeaders: map[string]string{},+ }+ return c, nil+}++// restAPICredentialsFromConfig extracts and normalizes stored credentials.+func restAPICredentialsFromConfig(config map[string]any) map[string]any {+ raw, ok := config["credentials"]+ if !ok || raw == nil {+ return map[string]any{}+ }+ if creds, ok := raw.(map[string]any); ok {+ return creds+ }+ if text, ok := raw.(string); ok {+ var creds map[string]any+ if json.Unmarshal([]byte(text), &creds) == nil && creds != nil {+ return creds+ }+ }+ return map[string]any{}+}++// parseRestAPIConfig mirrors RestAPIConnectorConfig + parse_storage_config.+func parseRestAPIConfig(config map[string]any) (*restAPIConfig, error) {+ cfg := &restAPIConfig{+ Method: "GET",+ AuthType: restAPIAuthNone,+ PaginationType: restAPIPaginationNone,+ BatchSize: restAPIDefaultBatchSize,+ MaxPages: restAPIDefaultMaxPages,+ RequestDelay: restAPIDefaultRequestDelay,+ }++ cfg.URL = strings.TrimSpace(stringConfig(config["url"]))+ if cfg.URL == "" {+ return nil, &ConnectorValidationError{Message: "Invalid REST API config: url\n Field required"}+ }+ if method := strings.ToUpper(strings.TrimSpace(stringConfig(config["method"]))); method != "" {+ cfg.Method = method+ }+ if cfg.Method != "GET" && cfg.Method != "POST" {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported HTTP method '%s'.", cfg.Method)}+ }+ cfg.Headers = restAPITextToDict(config["headers"])+ cfg.QueryParams = restAPITextToDict(config["query_params"])++ if auth := strings.TrimSpace(stringConfig(config["auth_type"])); auth != "" {+ cfg.AuthType = auth+ }+ switch cfg.AuthType {+ case restAPIAuthNone, restAPIAuthAPIKeyHeader, restAPIAuthBearer, restAPIAuthBasic:+ default:+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported auth_type '%s'.", cfg.AuthType)}+ }+ cfg.AuthConfig = configAnyMap(config["auth_config"])+ cfg.ItemsPath = strings.TrimSpace(stringConfig(config["items_path"]))+ cfg.IDField = strings.TrimSpace(stringConfig(config["id_field"]))+ cfg.ContentFields = restAPICoerceFieldList(config["content_fields"])+ if len(cfg.ContentFields) == 0 {+ return nil, &ConnectorValidationError{Message: "At least one content field must be configured (content_fields)."}+ }+ cfg.MetadataFields = restAPICoerceFieldList(config["metadata_fields"])++ if pagination := strings.TrimSpace(stringConfig(config["pagination_type"])); pagination != "" {+ cfg.PaginationType = pagination+ }+ switch cfg.PaginationType {+ case restAPIPaginationNone, restAPIPaginationPage, restAPIPaginationOffset, restAPIPaginationCursor:+ default:+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported pagination_type '%s'.", cfg.PaginationType)}+ }+ cfg.PaginationConfig = configAnyMap(config["pagination_config"])+ cfg.PollTimestampField = strings.TrimSpace(stringConfig(config["poll_timestamp_field"]))++ cfg.RequestBody = configAnyMap(config["request_body"])+ if len(cfg.RequestBody) == 0 {+ cfg.RequestBody = configAnyMap(cfg.PaginationConfig["request_body"])+ }+ cfg.FieldTypeHints = configStringMap(config["field_type_hints"])+ cfg.FieldDefaultValues = configAnyMap(config["field_default_values"])+ cfg.ContentTemplate = stringConfig(config["content_template"])+ if v, ok := restAPIConfigInt(config["batch_size"]); ok {+ cfg.BatchSize = v+ }+ if cfg.BatchSize <= 0 {+ cfg.BatchSize = restAPIDefaultBatchSize+ }+ if v, ok := restAPIConfigInt(config["max_pages"]); ok {+ if v <= 0 {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Invalid REST API config: max_pages must be a positive integer, got %d", v)}+ }+ cfg.MaxPages = v+ }+ if v := restAPIConfigFloat(config["request_delay"]); v >= 0 {+ cfg.RequestDelay = v+ }+ return cfg, nil+}++// restAPITextToDict mirrors _text_to_dict: dict, JSON string, or key=value+// lines are all accepted.+func restAPITextToDict(value any) map[string]string {+ if value == nil {+ return nil+ }+ if dict, ok := value.(map[string]any); ok {+ out := make(map[string]string, len(dict))+ for k, v := range dict {+ out[k] = fmt.Sprint(v)+ }+ return out+ }+ text, ok := value.(string)+ if !ok || strings.TrimSpace(text) == "" {+ return nil+ }+ var parsed map[string]any+ if json.Unmarshal([]byte(text), &parsed) == nil && parsed != nil {+ out := make(map[string]string, len(parsed))+ for k, v := range parsed {+ out[k] = fmt.Sprint(v)+ }+ return out+ }+ out := map[string]string{}+ for _, line := range strings.Split(text, "\n") {+ line = strings.TrimSpace(line)+ if line == "" || strings.HasPrefix(line, "#") {+ continue+ }+ if idx := strings.Index(line, "="); idx >= 0 {+ out[strings.TrimSpace(line[:idx])] = strings.TrimSpace(line[idx+1:])+ }+ }+ return out+}++// restAPICoerceFieldList mirrors the content_fields/metadata_fields coercion.+func restAPICoerceFieldList(value any) []string {+ if value == nil {+ return nil+ }+ if text, ok := value.(string); ok {+ text = strings.TrimSpace(text)+ if text == "" {+ return nil+ }+ parts := strings.Split(text, ",")+ out := make([]string, 0, len(parts))+ for _, p := range parts {+ if p = strings.TrimSpace(p); p != "" {+ out = append(out, p)+ }+ }+ return out+ }+ if list, ok := value.([]any); ok {+ out := make([]string, 0, len(list))+ for _, item := range list {+ if s := strings.TrimSpace(fmt.Sprint(item)); s != "" {+ out = append(out, s)+ }+ }+ return out+ }+ return nil+}++func configAnyMap(value any) map[string]any {+ if m, ok := value.(map[string]any); ok {+ return m+ }+ return nil+}++func configStringMap(value any) map[string]string {+ if m, ok := value.(map[string]any); ok {+ out := make(map[string]string, len(m))+ for k, v := range m {+ out[k] = stringConfig(v)+ }+ return out+ }+ return nil+}++// restAPIConfigInt reads an integer config value without the positive-only+// coercion of configInt so values like max_pages=0 round-trip faithfully.+func restAPIConfigInt(value any) (int, bool) {+ switch typed := value.(type) {+ case int:+ return typed, true+ case int64:+ return int(typed), true+ case float64:+ return int(typed), true+ case json.Number:+ if v, err := typed.Int64(); err == nil {+ return int(v), true+ }+ case string:+ if v, err := strconv.Atoi(strings.TrimSpace(typed)); err == nil {+ return v, true+ }+ }+ return 0, false+}++func restAPIConfigFloat(value any) float64 {+ switch typed := value.(type) {+ case float64:+ return typed+ case int:+ return float64(typed)+ case int64:+ return float64(typed)+ case json.Number:+ if v, err := typed.Float64(); err == nil {+ return v+ }+ case string:+ if v, err := strconv.ParseFloat(strings.TrimSpace(typed), 64); err == nil {+ return v+ }+ }+ return -1+}++// restAPIURLQueryParams splits the URL query string into a string map, keeping+// the last value per key.+func restAPIURLQueryParams(parsed *url.URL) map[string]string {+ if parsed.RawQuery == "" {+ return nil+ }+ values := parsed.Query()+ out := make(map[string]string, len(values))+ for k, list := range values {+ if len(list) > 0 {+ out[k] = list[len(list)-1]+ }+ }+ return out+}++// prepare builds auth headers from credentials, mirroring _build_auth.+func (c *RestAPIConnector) prepare() error {+ if c.prepared {+ return nil+ }+ c.authHeaders = map[string]string{}+ c.basicAuth = nil++ switch c.cfg.AuthType {+ case restAPIAuthNone:+ case restAPIAuthAPIKeyHeader:+ headerName := strings.TrimSpace(stringConfig(c.cfg.AuthConfig["header_name"]))+ apiKey := stringConfig(c.credentials["api_key"])+ if apiKey == "" {+ apiKey = stringConfig(c.cfg.AuthConfig["api_key_value"])+ }+ if apiKey == "" {+ apiKey = stringConfig(c.cfg.AuthConfig["api_key"])+ }+ if headerName == "" || apiKey == "" {+ return &ConnectorMissingCredentialError{Message: "REST API (api_key_header) requires 'header_name' in auth_config and 'api_key' in credentials"}+ }+ c.authHeaders[headerName] = apiKey+ case restAPIAuthBearer:+ token := stringConfig(c.credentials["token"])+ if token == "" {+ token = stringConfig(c.cfg.AuthConfig["token"])+ }+ if token == "" {+ return &ConnectorMissingCredentialError{Message: "REST API (bearer) requires 'token' in credentials"}+ }+ c.authHeaders["Authorization"] = "Bearer " + token+ case restAPIAuthBasic:+ username := stringConfig(c.credentials["username"])+ if username == "" {+ username = stringConfig(c.cfg.AuthConfig["username"])+ }+ password := stringConfig(c.credentials["password"])+ if password == "" {+ password = stringConfig(c.cfg.AuthConfig["password"])+ }+ if username == "" || password == "" {+ return &ConnectorMissingCredentialError{Message: "REST API (basic) requires 'username' and 'password'"}+ }+ c.basicAuth = &restAPIBasicAuth{username: username, password: password}+ default:+ return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported auth_type: %s", c.cfg.AuthType)}+ }+ c.prepared = true+ return nil+}++// Validate validates configuration and credential presence without network+// I/O; live connectivity validation is performed by ValidateLive.+func (c *RestAPIConnector) Validate(ctx context.Context) error {+ if c == nil {+ return &ConnectorValidationError{Message: "rest API connector is nil"}+ }+ return c.prepare()+}++// ValidateLive mirrors RestAPIConnector.validate_config: schema validation plus+// a live single-page fetch with the stored credentials.+func (c *RestAPIConnector) ValidateLive(ctx context.Context) error {+ if c == nil {+ return &ConnectorValidationError{Message: "rest API connector is nil"}+ }+ if err := c.prepare(); err != nil {+ return err+ }+ params := map[string]any{}+ switch c.cfg.PaginationType {+ case restAPIPaginationPage:+ page := 1+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["start_page"]); ok {+ page = v+ }+ perPage, err := c.resolvePageSize()+ if err != nil {+ return err+ }+ c.applyPagePagination(params, page, perPage)+ case restAPIPaginationOffset:+ perPage, err := c.resolvePageSize()+ if err != nil {+ return err+ }+ offset := 0+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["start_offset"]); ok {+ offset = v+ }+ limit := perPage+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["limit"]); ok {+ limit = v+ }+ if limit <= 0 {+ limit = perPage+ }+ c.applyOffsetPagination(params, offset, limit)+ case restAPIPaginationCursor:+ if cursor := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["initial_cursor"])); cursor != "" {+ c.applyCursorPagination(params, cursor)+ }+ }+ _, err := c.fetchPage(ctx, params)+ return err+}++// OpenSync opens one sync session. Incremental runs filter documents to the+// request window (start <= updated_at < end).+func (c *RestAPIConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.prepare(); err != nil {+ return nil, err+ }+ var windowStart *time.Time+ if !request.FromBeginning && request.WindowStart != nil {+ ws := request.WindowStart.UTC()+ windowStart = &ws+ }+ session := &restAPISyncSession{+ connector: c,+ iterator: newRestAPIItemIterator(c),+ batchSize: c.cfg.BatchSize,+ fromBeginning: request.FromBeginning,+ windowStart: windowStart,+ windowEnd: request.WindowEnd.UTC(),+ }+ session.applyResume(request.Resume)+ return session, nil+}++// OpenPrune reports prune as unsupported for sources that cannot enumerate a+// complete slim snapshot; prune tasks then complete as no-ops.+func (c *RestAPIConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ return nil, ErrPruneUnsupported+}++// ---------------------------------------------------------------------------+// HTTP fetching+// ---------------------------------------------------------------------------++// fetchPage fetches a single page with retry, mirroring _fetch_page.+func (c *RestAPIConnector) fetchPage(ctx context.Context, params map[string]any) (any, error) {+ delay := restAPIRetryBaseDelay+ var lastErr error+ for attempt := 0; attempt < restAPIRetryTries; attempt++ {+ response, err := c.fetchPageOnce(ctx, params)+ if err == nil {+ return response, nil+ }+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }+ var (+ valErr *ConnectorValidationError+ credErr *ConnectorMissingCredentialError+ rateErr *RateLimitTriedTooManyTimesError+ )+ if errors.As(err, &valErr) || errors.As(err, &credErr) || errors.As(err, &rateErr) {+ return nil, err+ }+ lastErr = err+ if attempt == restAPIRetryTries-1 {+ break+ }+ sleepFor := delay + time.Duration(rand.Float64()*float64(restAPIRetryJitter))+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case <-time.After(sleepFor):+ }+ delay = time.Duration(float64(delay) * float64(restAPIRetryBackoff))+ if delay > restAPIRetryMaxDelay {+ delay = restAPIRetryMaxDelay+ }+ }+ return nil, lastErr+}++// fetchPageOnce performs one page fetch including the 429 wait loop and manual+// redirect handling with per-hop SSRF validation and DNS pinning.+func (c *RestAPIConnector) fetchPageOnce(ctx context.Context, params map[string]any) (any, error) {+ if err := c.prepare(); err != nil {+ return nil, err+ }+ merged := map[string]any{}+ for k, v := range c.urlParams {+ merged[k] = v+ }+ for k, v := range c.cfg.QueryParams {+ merged[k] = v+ }+ for k, v := range params {+ merged[k] = v+ }++ currentURL, queryParams := c.buildURLWithTemplates(merged)+ currentMethod := c.cfg.Method+ currentHeaders := make(map[string]string, len(c.cfg.Headers)+len(c.authHeaders))+ for k, v := range c.cfg.Headers {+ currentHeaders[k] = v+ }+ for k, v := range c.authHeaders {+ currentHeaders[k] = v+ }+ currentBody := c.cfg.RequestBody+ currentAuth := c.basicAuth+ previousNetloc := restAPINetloc(currentURL)++ for hop := 0; hop <= restAPIMaxRedirects; hop++ {+ hostname, pinIP, err := assertRestAPIURLSafe(ctx, currentURL)+ if err != nil {+ return nil, &ConnectorValidationError{Message: "Unsafe REST API URL: " + err.Error()}+ }++ var resp *http.Response+ var respErr error+ for wait := 0; wait < restAPI429MaxWaits; wait++ {+ resp, respErr = c.doRequest(ctx, currentMethod, currentURL, currentHeaders, queryParams, currentBody, currentAuth, hostname, pinIP)+ if respErr != nil {+ return nil, respErr+ }+ if resp.StatusCode != http.StatusTooManyRequests {+ break+ }+ retryAfter := restAPI429DefaultWait+ if raw := resp.Header.Get("Retry-After"); raw != "" {+ if seconds, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && seconds >= 0 {+ retryAfter = time.Duration(seconds) * time.Second+ }+ }+ resp.Body.Close()+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case <-time.After(retryAfter):+ }+ }+ if respErr != nil {+ return nil, respErr+ }+ if resp.StatusCode == http.StatusTooManyRequests {+ resp.Body.Close()+ return nil, &RateLimitTriedTooManyTimesError{Message: fmt.Sprintf("REST API rate limited: exceeded '%d' retries (too many requests)", restAPI429MaxWaits)}+ }++ if restAPIIsRedirect(resp.StatusCode) {+ location := resp.Header.Get("Location")+ if location == "" {+ return c.handleResponse(resp)+ }+ nextURL, err := restAPIResolveURL(currentURL, location)+ if err != nil {+ resp.Body.Close()+ return nil, &ConnectorValidationError{Message: "Unsafe REST API URL: " + err.Error()}+ }+ nextNetloc := restAPINetloc(nextURL)+ if nextNetloc != "" && nextNetloc != previousNetloc {+ currentHeaders = restAPIStripAuthHeaders(currentHeaders)+ currentAuth = nil+ }+ previousNetloc = nextNetloc+ if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusSeeOther {+ currentMethod = "GET"+ currentBody = nil+ queryParams = nil+ }+ resp.Body.Close()+ currentURL = nextURL+ continue+ }+ return c.handleResponse(resp)+ }+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Exceeded %d redirects fetching %q", restAPIMaxRedirects, currentURL)}+}++func (c *RestAPIConnector) doRequest(+ ctx context.Context,+ method, rawURL string,+ headers map[string]string,+ params map[string]any,+ body map[string]any,+ auth *restAPIBasicAuth,+ hostname string,+ pinIP net.IP,+) (*http.Response, error) {+ transport := newRestAPIPinnedTransport(hostname, pinIP)+ client := &http.Client{+ Transport: transport,+ Timeout: restAPIRequestTimeout,+ // Redirects are handled manually so auth headers can be stripped on+ // cross-origin hops and each hop is re-validated against SSRF.+ CheckRedirect: func(req *http.Request, via []*http.Request) error {+ return http.ErrUseLastResponse+ },+ }++ fullURL := rawURL+ if encoded := restAPIEncodeParams(params); encoded != "" {+ sep := "?"+ if strings.Contains(rawURL, "?") {+ sep = "&"+ }+ fullURL = rawURL + sep + encoded+ }++ var req *http.Request+ var err error+ switch method {+ case "GET":+ req, err = http.NewRequestWithContext(ctx, http.MethodGet, fullURL, nil)+ case "POST":+ var payload []byte+ if body == nil {+ payload = []byte("{}")+ } else {+ payload, err = json.Marshal(body)+ if err != nil {+ return nil, &ConnectorValidationError{Message: "REST API request body is not valid JSON"}+ }+ }+ req, err = http.NewRequestWithContext(ctx, http.MethodPost, fullURL, strings.NewReader(string(payload)))+ if err == nil {+ req.Header.Set("Content-Type", "application/json")+ }+ default:+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported HTTP method: %s", method)}+ }+ if err != nil {+ return nil, err+ }+ for k, v := range headers {+ req.Header.Set(k, v)+ }+ if auth != nil {+ req.SetBasicAuth(auth.username, auth.password)+ }+ resp, err := client.Do(req)+ if err != nil {+ transport.CloseIdleConnections()+ return nil, err+ }+ resp.Body = &restAPICloseIdleBody{body: resp.Body, transport: transport}+ return resp, nil+}++// restAPICloseIdleBody closes the underlying response body and then releases+// the pinned transport's idle connections, so the per-request transports do+// not leak keep-alive sockets during long paginated syncs.+type restAPICloseIdleBody struct {+ body io.ReadCloser+ transport *http.Transport+}++func (b *restAPICloseIdleBody) Read(p []byte) (int, error) { return b.body.Read(p) }++func (b *restAPICloseIdleBody) Close() error {+ err := b.body.Close()+ b.transport.CloseIdleConnections()+ return err+}++func (c *RestAPIConnector) handleResponse(resp *http.Response) (any, error) {+ defer resp.Body.Close()+ status := resp.StatusCode+ switch {+ case status == http.StatusUnauthorized || status == http.StatusForbidden:+ return nil, &ConnectorMissingCredentialError{Message: fmt.Sprintf("REST API authentication failed with status %d", status)}+ case status >= 400 && status < 500 && status != http.StatusTooManyRequests:+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("REST API request failed with non-retriable client error status %d", status)}+ case status >= 500:+ // The "http <status>" wording lets the syncer's task-level retry+ // classifier (isTransientSyncError) recognize exhausted server errors.+ return nil, fmt.Errorf("REST API request failed with http %d", status)+ }++ raw, err := io.ReadAll(resp.Body)+ if err != nil {+ return nil, err+ }+ decoder := json.NewDecoder(strings.NewReader(string(raw)))+ decoder.UseNumber()+ var value any+ if err := decoder.Decode(&value); err != nil {+ return nil, &ConnectorValidationError{Message: "REST API response is not valid JSON"}+ }+ return value, nil+}++// newRestAPIPinnedTransport resolves the target once (in assertRestAPIURLSafe)+// and pins that address for the actual connection, preventing DNS rebinding.+func newRestAPIPinnedTransport(hostname string, pinIP net.IP) *http.Transport {+ dialer := &net.Dialer{Timeout: restAPIRequestTimeout, KeepAlive: 30 * time.Second}+ return &http.Transport{+ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {+ _, port, err := net.SplitHostPort(addr)+ if err != nil {+ port = "443"+ }+ return dialer.DialContext(ctx, network, net.JoinHostPort(pinIP.String(), port))+ },+ TLSClientConfig: &tls.Config{+ ServerName: hostname,+ MinVersion: tls.VersionTLS12,+ },+ ForceAttemptHTTP2: true,+ }+}++// buildURLWithTemplates substitutes {key} placeholders in the URL and returns+// the remaining query parameters.+func (c *RestAPIConnector) buildURLWithTemplates(params map[string]any) (string, map[string]any) {+ u := c.baseURL+ remaining := make(map[string]any, len(params))+ for k, v := range params {+ remaining[k] = v+ }+ for k, v := range params {+ placeholder := "{" + k + "}"+ if strings.Contains(u, placeholder) {+ u = strings.ReplaceAll(u, placeholder, restAPIParamString(v))+ delete(remaining, k)+ }+ }+ return u, remaining+}++func restAPIParamString(value any) string {+ switch typed := value.(type) {+ case string:+ return typed+ case json.Number:+ return typed.String()+ case bool:+ if typed {+ return "True"+ }+ return "False"+ case nil:+ return ""+ default:+ return fmt.Sprint(value)+ }+}++func restAPIEncodeParams(params map[string]any) string {+ if len(params) == 0 {+ return ""+ }+ values := url.Values{}+ for k, v := range params {+ values.Set(k, restAPIParamString(v))+ }+ return values.Encode()+}++func restAPINetloc(rawURL string) string {+ parsed, err := url.Parse(rawURL)+ if err != nil {+ return ""+ }+ return parsed.Host+}++func restAPIResolveURL(base, location string) (string, error) {+ baseParsed, err := url.Parse(base)+ if err != nil {+ return "", err+ }+ ref, err := url.Parse(location)+ if err != nil {+ return "", err+ }+ return baseParsed.ResolveReference(ref).String(), nil+}++func restAPIIsRedirect(status int) bool {+ switch status {+ case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,+ http.StatusTemporaryRedirect, http.StatusPermanentRedirect:+ return true+ }+ return false+}++var restAPIAuthSensitiveHeaders = map[string]struct{}{+ "authorization": {},+ "proxy-authorization": {},+ "apikey": {},+ "api-key": {},+ "x-api-key": {},+ "x-auth-token": {},+}++func restAPIStripAuthHeaders(headers map[string]string) map[string]string {+ out := make(map[string]string, len(headers))+ for k, v := range headers {+ if _, sensitive := restAPIAuthSensitiveHeaders[strings.ToLower(k)]; sensitive {+ continue+ }+ out[k] = v+ }+ return out+}++// ---------------------------------------------------------------------------+// SSRF protection+// ---------------------------------------------------------------------------++// validateRestAPIURLForSSRF performs quick deny-list checks plus DNS+// resolution. Resolution failure is logged and tolerated because the+// per-request check re-validates.+func validateRestAPIURLForSSRF(rawURL string) error {+ parsed, err := url.Parse(rawURL)+ if err != nil {+ return &ConnectorValidationError{Message: "REST API connector URL must include a hostname."}+ }+ if parsed.Scheme != "http" && parsed.Scheme != "https" {+ return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported URL scheme for REST API connector: %q. Only http/https are allowed.", parsed.Scheme)}+ }+ hostname := parsed.Hostname()+ if hostname == "" {+ return &ConnectorValidationError{Message: "REST API connector URL must include a hostname."}+ }+ if strings.EqualFold(hostname, "localhost") {+ return &ConnectorValidationError{Message: fmt.Sprintf("REST API connector URL hostname %q is not allowed (localhost is blocked).", hostname)}+ }+ if restAPISSRFAllowLoopback {+ return nil+ }+ addrs, err := net.LookupIP(hostname)+ if err != nil {+ // DNS failure is not an SSRF condition by itself; the per-request+ // check will surface it if it matters.+ return nil+ }+ for _, addr := range addrs {+ if !restAPIIPIsGlobal(restAPIEffectiveIP(addr)) {+ return &ConnectorValidationError{Message: fmt.Sprintf(+ "REST API connector URL %q resolves to disallowed address %s (localhost, private, link-local, reserved, or multicast addresses are blocked).",+ rawURL, addr)}+ }+ }+ return nil+}++// assertRestAPIURLSafe mirrors ssrf_guard.assert_url_is_safe: every resolved+// address must be globally routable. It returns the hostname and the first+// validated IP so the caller can pin DNS.+func assertRestAPIURLSafe(ctx context.Context, rawURL string) (string, net.IP, error) {+ parsed, err := url.Parse(rawURL)+ if err != nil {+ return "", nil, fmt.Errorf("URL is missing a host.")+ }+ if parsed.Scheme != "http" && parsed.Scheme != "https" {+ return "", nil, fmt.Errorf("Disallowed URL scheme: %q. Only [http https] are allowed.", parsed.Scheme)+ }+ hostname := parsed.Hostname()+ if hostname == "" {+ return "", nil, fmt.Errorf("URL is missing a host.")+ }+ if restAPISSRFAllowLoopback {+ addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)+ if err != nil {+ return "", nil, fmt.Errorf("Could not resolve hostname %q: %v", hostname, err)+ }+ if len(addrs) == 0 {+ return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)+ }+ return hostname, addrs[0].IP, nil+ }++ addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)+ if err != nil {+ return "", nil, fmt.Errorf("Could not resolve hostname %q: %v", hostname, err)+ }+ var first net.IP+ for _, addr := range addrs {+ eff := restAPIEffectiveIP(addr.IP)+ if !restAPIIPIsGlobal(eff) {+ return "", nil, fmt.Errorf("URL resolves to a non-public address (%s), which is not allowed.", addr.IP)+ }+ if first == nil {+ first = addr.IP+ }+ }+ if first == nil {+ return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)+ }+ return hostname, first, nil+}++// restAPIEffectiveIP returns the IPv4 equivalent for IPv4-mapped IPv6+// addresses, mirroring ssrf_guard._effective_ip.+func restAPIEffectiveIP(ip net.IP) net.IP {+ if v4 := ip.To4(); v4 != nil && len(ip) == net.IPv6len {+ return v4+ }+ return ip+}++// restAPIIPIsGlobal mirrors ipaddress.is_global for the address classes that+// matter in practice.+func restAPIIPIsGlobal(ip net.IP) bool {+ if ip == nil {+ return false+ }+ if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() || ip.IsMulticast() ||+ ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() {+ return false+ }+ if v4 := ip.To4(); v4 != nil {+ first := v4[0]+ // 0.0.0.0/8, 100.64.0.0/10 (CGNAT), 192.0.0.0/24 (IETF protocol+ // assignments), 198.18.0.0/15 (benchmarking), 240.0.0.0/4, broadcast.+ if first == 0 || first == 100 || (first == 192 && v4[1] == 0) ||+ (first == 198 && v4[1]&0xfe == 18) || first >= 240 {+ return false+ }+ return true+ }+ // IPv6 documentation range (2001:db8::/32) is not globally routable.+ if len(ip) == net.IPv6len && ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x0d && ip[3] == 0xb8 {+ return false+ }+ return true+}++// ---------------------------------------------------------------------------+// Pagination+// ---------------------------------------------------------------------------++// resolvePageSize mirrors _resolve_page_size.+func (c *RestAPIConnector) resolvePageSize() (int, error) {+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["page_size"]); ok && v > 0 {+ return v, nil+ }+ sizeParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["page_size_param"]))+ if sizeParam == "" {+ sizeParam = strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["limit_param"]))+ }+ if sizeParam != "" {+ for _, source := range []map[string]string{c.cfg.QueryParams, c.urlParams} {+ if raw, ok := source[sizeParam]; ok {+ if v, err := strconv.Atoi(raw); err == nil && v > 0 {+ return v, nil+ }+ }+ }+ }+ return c.cfg.BatchSize, nil+}++func (c *RestAPIConnector) applyPagePagination(params map[string]any, page, perPage int) {+ pageParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["page_param"]))+ if pageParam == "" {+ pageParam = "page"+ }+ params[pageParam] = page+ if sizeParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["page_size_param"])); sizeParam != "" {+ params[sizeParam] = perPage+ }+}++func (c *RestAPIConnector) applyOffsetPagination(params map[string]any, offset, limit int) {+ offsetParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["offset_param"]))+ if offsetParam == "" {+ offsetParam = "offset"+ }+ params[offsetParam] = offset+ if limitParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["limit_param"])); limitParam != "" {+ params[limitParam] = limit+ }+}++func (c *RestAPIConnector) applyCursorPagination(params map[string]any, cursor string) {+ cursorParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["cursor_param"]))+ if cursorParam == "" {+ cursorParam = "cursor"+ }+ params[cursorParam] = cursor+}++// restAPIItemIterator mirrors _iter_items: it walks pages applying the+// configured pagination and stopping when the source reports no more items.+type restAPIItemIterator struct {+ c *RestAPIConnector+ pageCount int+ page int+ offset int+ limit int+ perPage int+ cursor string+ finished bool+}++func newRestAPIItemIterator(c *RestAPIConnector) *restAPIItemIterator {+ it := &restAPIItemIterator{c: c}+ if perPage, err := c.resolvePageSize(); err == nil {+ it.perPage = perPage+ } else {+ it.perPage = c.cfg.BatchSize+ }+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["start_page"]); ok {+ it.page = v+ } else {+ it.page = 1+ }+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["start_offset"]); ok {+ it.offset = v+ }+ it.limit = it.perPage+ if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["limit"]); ok {+ it.limit = v+ }+ if it.limit <= 0 {+ it.limit = it.perPage+ }+ it.cursor = strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["initial_cursor"]))+ return it+}++// nextPage returns the items of the next page, or nil when iteration is done.+func (it *restAPIItemIterator) nextPage(ctx context.Context) ([]map[string]any, error) {+ if it.finished || it.pageCount >= it.c.cfg.MaxPages {+ return nil, nil+ }+ params := map[string]any{}+ switch it.c.cfg.PaginationType {+ case restAPIPaginationPage:+ it.c.applyPagePagination(params, it.page, it.perPage)+ case restAPIPaginationOffset:+ it.c.applyOffsetPagination(params, it.offset, it.limit)+ case restAPIPaginationCursor:+ if it.cursor != "" {+ it.c.applyCursorPagination(params, it.cursor)+ }+ }+ if it.pageCount > 0 && it.c.cfg.RequestDelay > 0 {+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case <-time.After(time.Duration(it.c.cfg.RequestDelay * float64(time.Second))):+ }+ }++ response, err := it.c.fetchPage(ctx, params)+ if err != nil {+ var (+ valErr *ConnectorValidationError+ credErr *ConnectorMissingCredentialError+ )+ if errors.As(err, &valErr) || errors.As(err, &credErr) {+ return nil, err+ }+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("REST API page fetch failed: %v", err)}+ }++ items := restAPIExtractItems(response, it.c.cfg.ItemsPath)+ if len(items) == 0 {+ it.finished = true+ return nil, nil+ }+ it.pageCount++++ switch it.c.cfg.PaginationType {+ case restAPIPaginationNone:+ it.finished = true+ case restAPIPaginationPage:+ if len(items) < it.perPage {+ it.finished = true+ } else {+ it.page+++ }+ case restAPIPaginationOffset:+ if len(items) < it.limit {+ it.finished = true+ } else {+ it.offset += it.limit+ }+ case restAPIPaginationCursor:+ next := restAPIExtractNextCursor(response, it.c.cfg.PaginationConfig)+ if next == "" {+ it.finished = true+ } else {+ it.cursor = next+ }+ }+ return items, nil+}++// positionAfterPage returns the resume position after the page that was just+// consumed, i.e. the next page to fetch. It is nil for pagination_type=none,+// which never emits checkpoints.+func (it *restAPIItemIterator) positionAfterPage() *restAPISyncCursor {+ switch it.c.cfg.PaginationType {+ case restAPIPaginationPage:+ return &restAPISyncCursor{Page: it.page}+ case restAPIPaginationOffset:+ return &restAPISyncCursor{Offset: it.offset}+ case restAPIPaginationCursor:+ return &restAPISyncCursor{Cursor: it.cursor}+ }+ return nil+}++// ---------------------------------------------------------------------------+// JSON extraction+// ---------------------------------------------------------------------------++// restAPIExtractItems mirrors _extract_items.+func restAPIExtractItems(response any, itemsPath string) []map[string]any {+ var items []any+ if itemsPath != "" {+ matches, err := restAPIJSONPathMatches(response, itemsPath)+ if err != nil {+ return nil+ }+ if len(matches) == 0 {+ return nil+ }+ if len(matches) == 1 {+ if list, ok := matches[0].([]any); ok {+ items = list+ } else {+ items = matches+ }+ } else {+ items = matches+ }+ } else if list, ok := response.([]any); ok {+ items = list+ } else if dict, ok := response.(map[string]any); ok {+ for _, key := range []string{"items", "results", "data", "records"} {+ if value, ok := dict[key]; ok {+ if list, ok := value.([]any); ok {+ items = list+ break+ }+ }+ }+ if items == nil {+ for _, value := range dict {+ if list, ok := value.([]any); ok {+ items = list+ break+ }+ }+ }+ }++ out := make([]map[string]any, 0, len(items))+ for _, item := range items {+ if mapping, ok := item.(map[string]any); ok {+ out = append(out, mapping)+ }+ }+ return out+}++// restAPIExtractNextCursor mirrors _extract_next_cursor.+func restAPIExtractNextCursor(response any, paginationConfig map[string]any) string {+ cursorPath := strings.TrimSpace(stringConfig(paginationConfig["next_cursor_path"]))+ if cursorPath == "" {+ field := strings.TrimSpace(stringConfig(paginationConfig["next_cursor_field"]))+ if field != "" {+ if dict, ok := response.(map[string]any); ok {+ if value, ok := dict[field]; ok && value != nil {+ return fmt.Sprint(value)+ }+ }+ }+ return ""+ }+ matches, err := restAPIJSONPathMatches(response, cursorPath)+ if err != nil || len(matches) == 0 || matches[0] == nil {+ return ""+ }+ return fmt.Sprint(matches[0])+}++const (+ restAPIJSONPathField = iota+ restAPIJSONPathIndex+ restAPIJSONPathWildcard+)++type restAPIJSONPathToken struct {+ kind int+ key string+ index int+ recursive bool+}++// restAPIJSONPathMatches implements the commonly used subset of jsonpath: $,+// dot keys, [n], [*], and .. recursive descent.+func restAPIJSONPathMatches(root any, path string) ([]any, error) {+ tokens, err := parseRestAPIJSONPath(path)+ if err != nil {+ return nil, err+ }+ current := []any{root}+ for _, token := range tokens {+ next := []any{}+ for _, value := range current {+ if token.recursive {+ restAPIJSONPathCollectRecursive(value, token, &next)+ continue+ }+ switch token.kind {+ case restAPIJSONPathField:+ if dict, ok := value.(map[string]any); ok {+ if child, ok := dict[token.key]; ok {+ next = append(next, child)+ }+ }+ case restAPIJSONPathIndex:+ if list, ok := value.([]any); ok && token.index >= 0 && token.index < len(list) {+ next = append(next, list[token.index])+ }+ case restAPIJSONPathWildcard:+ switch typed := value.(type) {+ case []any:+ next = append(next, typed...)+ case map[string]any:+ for _, child := range typed {+ next = append(next, child)+ }+ }+ }+ }+ current = next+ if len(current) == 0 {+ break+ }+ }+ return current, nil+}++func restAPIJSONPathCollectRecursive(value any, token restAPIJSONPathToken, out *[]any) {+ if token.kind != restAPIJSONPathField {+ return+ }+ if dict, ok := value.(map[string]any); ok {+ if child, ok := dict[token.key]; ok {+ *out = append(*out, child)+ }+ for _, nested := range dict {+ restAPIJSONPathCollectRecursive(nested, token, out)+ }+ }+ if list, ok := value.([]any); ok {+ for _, nested := range list {+ restAPIJSONPathCollectRecursive(nested, token, out)+ }+ }+}++func parseRestAPIJSONPath(path string) ([]restAPIJSONPathToken, error) {+ s := strings.TrimSpace(path)+ if strings.HasPrefix(s, "$") {+ s = s[1:]+ }+ var tokens []restAPIJSONPathToken+ pendingRecursive := false+ i := 0+ for i < len(s) {+ switch s[i] {+ case '.':+ if i+1 < len(s) && s[i+1] == '.' {+ pendingRecursive = true+ i += 2+ } else {+ i+++ }+ case '[':+ end := strings.IndexByte(s[i:], ']')+ if end < 0 {+ return nil, fmt.Errorf("unterminated bracket in JSONPath")+ }+ content := strings.TrimSpace(s[i+1 : i+end])+ token := restAPIJSONPathToken{recursive: pendingRecursive}+ pendingRecursive = false+ switch {+ case content == "*":+ token.kind = restAPIJSONPathWildcard+ default:+ if n, err := strconv.Atoi(content); err == nil {+ token.kind = restAPIJSONPathIndex+ token.index = n+ } else {+ token.kind = restAPIJSONPathField+ token.key = strings.Trim(content, "'\"")+ }+ }+ tokens = append(tokens, token)+ i += end + 1+ default:+ start := i+ for i < len(s) && s[i] != '.' && s[i] != '[' {+ i+++ }+ key := s[start:i]+ if key == "" {+ return nil, fmt.Errorf("empty JSONPath segment")+ }+ tokens = append(tokens, restAPIJSONPathToken{kind: restAPIJSONPathField, key: key, recursive: pendingRecursive})+ pendingRecursive = false+ }+ }+ return tokens, nil+}++// ---------------------------------------------------------------------------+// Field extraction and item-to-document mapping+// ---------------------------------------------------------------------------++var restAPIFieldSegmentRE = regexp.MustCompile(`^([^\[\]]+)(\[(\d+|\*)\])?$`)++// extractRestAPIFieldValues mirrors _extract_field_values: dot-notation with+// optional [index] / [*] segments.+func extractRestAPIFieldValues(item map[string]any, path string) []any {+ if path == "" {+ return nil+ }+ current := []any{item}+ for _, segment := range strings.Split(path, ".") {+ if segment == "" {+ return nil+ }+ match := restAPIFieldSegmentRE.FindStringSubmatch(segment)+ key := segment+ index := ""+ if match != nil {+ key = match[1]+ index = match[3]+ }+ next := []any{}+ for _, value := range current {+ dict, ok := value.(map[string]any)+ if !ok {+ continue+ }+ child, ok := dict[key]+ if !ok || child == nil {+ continue+ }+ if index == "" {+ next = append(next, child)+ continue+ }+ list, ok := child.([]any)+ if !ok {+ continue+ }+ if index == "*" {+ next = append(next, list...)+ continue+ }+ idx, err := strconv.Atoi(index)+ if err != nil {+ continue+ }+ if idx >= 0 && idx < len(list) {+ next = append(next, list[idx])+ }+ }+ current = next+ if len(current) == 0 {+ break+ }+ }+ return current+}++func extractRestAPIField(item map[string]any, path string) any {+ values := extractRestAPIFieldValues(item, path)+ if len(values) == 0 {+ return nil+ }+ if len(values) == 1 {+ return values[0]+ }+ return values+}++// getTypedFieldValue mirrors _get_typed_field_value.+func (c *RestAPIConnector) getTypedFieldValue(path string, item map[string]any) any {+ values := extractRestAPIFieldValues(item, path)+ if len(values) == 0 {+ return c.cfg.FieldDefaultValues[path]+ }+ hint := c.cfg.FieldTypeHints[path]+ converted := make([]any, 0, len(values))+ for _, value := range values {+ if cv := convertRestAPITypedValue(value, hint); cv != nil {+ converted = append(converted, cv)+ }+ }+ if len(converted) == 0 {+ return nil+ }+ if len(converted) == 1 {+ return converted[0]+ }+ parts := make([]string, 0, len(converted))+ for _, value := range converted {+ parts = append(parts, coerceRestAPIToText(value))+ }+ return strings.Join(parts, ", ")+}++func convertRestAPITypedValue(value any, hint string) any {+ switch hint {+ case "string":+ return restAPIValueText(value)+ case "number":+ f, ok := restAPIToFloat(value)+ if !ok {+ return nil+ }+ if f == math.Trunc(f) && !math.IsInf(f, 0) && math.Abs(f) < 1e15 {+ return int64(f)+ }+ return f+ case "date":+ if t, ok := value.(time.Time); ok {+ return restAPITimeISO(t)+ }+ if t := parseRestAPIDatetime(value); t != nil {+ return restAPITimeISO(*t)+ }+ return restAPIValueText(value)+ default:+ return value+ }+}++func restAPIToFloat(value any) (float64, bool) {+ switch typed := value.(type) {+ case json.Number:+ f, err := typed.Float64()+ return f, err == nil+ case float64:+ return typed, true+ case int:+ return float64(typed), true+ case int64:+ return float64(typed), true+ case string:+ f, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)+ return f, err == nil+ default:+ return 0, false+ }+}++// extractTimestamp mirrors _extract_timestamp.+func (c *RestAPIConnector) extractTimestamp(item map[string]any) *time.Time {+ if c.cfg.PollTimestampField == "" {+ return nil+ }+ value := extractRestAPIField(item, c.cfg.PollTimestampField)+ if list, ok := value.([]any); ok && len(list) > 0 {+ value = list[0]+ }+ return parseRestAPIDatetime(value)+}++// parseRestAPIDatetime mirrors _parse_datetime.+func parseRestAPIDatetime(value any) *time.Time {+ if value == nil {+ return nil+ }+ switch typed := value.(type) {+ case time.Time:+ t := typed.UTC()+ return &t+ case json.Number:+ if f, err := typed.Float64(); err == nil {+ return restAPITimestampFromFloat(f)+ }+ case float64:+ return restAPITimestampFromFloat(typed)+ case int:+ return restAPITimestampFromFloat(float64(typed))+ case int64:+ return restAPITimestampFromFloat(float64(typed))+ case string:+ text := strings.TrimSpace(typed)+ if text == "" {+ return nil+ }+ layouts := []string{+ "2006-01-02T15:04:05.999999999Z",+ "2006-01-02T15:04:05Z",+ "2006-01-02 15:04:05",+ "2006-01-02",+ }+ for _, layout := range layouts {+ if t, err := time.Parse(layout, text); err == nil {+ t = t.UTC()+ return &t+ }+ }+ normalized := strings.ReplaceAll(text, " ", "T")+ if strings.HasSuffix(normalized, "Z") {+ normalized = normalized[:len(normalized)-1] + "+00:00"+ }+ for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05.999999999-07:00", "2006-01-02T15:04:05-07:00"} {+ if t, err := time.Parse(layout, normalized); err == nil {+ t = t.UTC()+ return &t+ }+ }+ }+ return nil+}++func restAPITimestampFromFloat(f float64) *time.Time {+ sec := int64(f)+ nsec := int64((f - float64(sec)) * 1e9)+ t := time.Unix(sec, nsec).UTC()+ return &t+}++// itemToDocument mirrors _item_to_document.+func (c *RestAPIConnector) itemToDocument(item map[string]any) (SourceDocument, error) {+ var rawID string+ if c.cfg.IDField != "" {+ if v := c.getTypedFieldValue(c.cfg.IDField, item); v != nil {+ rawID = coerceRestAPIToText(v)+ } else {+ rawID = restAPIHash128("rest_api_item:" + restAPIStableText(item))+ }+ } else {+ rawID = restAPIHash128("rest_api_item:" + restAPIStableText(item))+ }+ docID := restAPIHash128("rest_api:" + rawID)++ var contentText string+ if c.cfg.ContentTemplate != "" {+ contentText = c.renderContentTemplate(item)+ } else {+ parts := []string{}+ for _, field := range c.cfg.ContentFields {+ if v := c.getTypedFieldValue(field, item); v != nil {+ if text := restAPIStripHTML(coerceRestAPIToText(v)); text != "" {+ parts = append(parts, text)+ }+ }+ }+ contentText = strings.Join(parts, "\n")+ }+ blob := []byte(contentText)++ var metadata map[string]any+ if len(c.cfg.MetadataFields) > 0 {+ metadata = map[string]any{}+ for _, field := range c.cfg.MetadataFields {+ if v := c.getTypedFieldValue(field, item); v != nil {+ metadata[field] = serializeRestAPIMetadataValue(v)+ }+ }+ }++ updatedAt := c.extractTimestamp(item)+ if updatedAt == nil {+ now := time.Now().UTC()+ updatedAt = &now+ }++ sem := rawID+ if len(c.cfg.ContentFields) > 0 {+ sem = restAPIValueText(extractRestAPIField(item, c.cfg.ContentFields[0]))+ }+ sem = restAPIStripHTML(sem)+ sem = strings.ReplaceAll(sem, "\n", " ")+ sem = strings.ReplaceAll(sem, "\r", " ")+ sem = strings.TrimSpace(sem)+ if len(sem) > 100 {+ sem = sem[:100]+ }+ if sem == "" {+ sem = docID+ }++ return SourceDocument{+ SourceID: docID,+ SemanticIdentifier: sem,+ Extension: ".txt",+ Blob: blob,+ UpdatedAt: *updatedAt,+ SizeBytes: int64(len(blob)),+ Metadata: metadata,+ Fingerprint: contentFingerprint(blob),+ }, nil+}++// renderContentTemplate mirrors _render_content_template.+func (c *RestAPIConnector) renderContentTemplate(item map[string]any) string {+ values := map[string]string{}+ seen := map[string]bool{}+ fields := append(append([]string{}, c.cfg.ContentFields...), c.cfg.MetadataFields...)+ for _, field := range fields {+ if seen[field] {+ continue+ }+ seen[field] = true+ if v := c.getTypedFieldValue(field, item); v != nil {+ values[restAPIFieldToTemplateName(field)] = coerceRestAPIToText(v)+ }+ }+ rendered, ok := renderRestAPITemplateSafeMap(c.cfg.ContentTemplate, values)+ if !ok {+ parts := []string{}+ for _, field := range c.cfg.ContentFields {+ if p := coerceRestAPIToText(c.getTypedFieldValue(field, item)); p != "" {+ parts = append(parts, p)+ }+ }+ rendered = strings.Join(parts, "\n")+ }+ return restAPIStripHTML(rendered)+}++var restAPITemplateIndexRE = regexp.MustCompile(`\[\d+\]|\[\*\]`)++func restAPIFieldToTemplateName(field string) string {+ name := restAPITemplateIndexRE.ReplaceAllString(field, "")+ return strings.ReplaceAll(name, ".", "_")+}++// renderRestAPITemplateSafeMap renders {name} placeholders from values with a+// safe lookup: unknown names become the empty string; malformed placeholders+// make the whole render fail so the caller falls back to joined content.+func renderRestAPITemplateSafeMap(template string, values map[string]string) (string, bool) {+ var b strings.Builder+ for i := 0; i < len(template); {+ switch template[i] {+ case '{':+ if i+1 < len(template) && template[i+1] == '{' {+ b.WriteByte('{')+ i += 2+ continue+ }+ end := strings.IndexByte(template[i:], '}')+ if end < 0 {+ return "", false+ }+ token := template[i+1 : i+end]+ if !validRestAPITemplateKey(token) {+ return "", false+ }+ b.WriteString(values[token])+ i += end + 1+ case '}':+ if i+1 < len(template) && template[i+1] == '}' {+ b.WriteByte('}')+ i += 2+ continue+ }+ return "", false+ default:+ b.WriteByte(template[i])+ i+++ }+ }+ return b.String(), true+}++func validRestAPITemplateKey(token string) bool {+ if token == "" {+ return false+ }+ for _, r := range token {+ if r != '_' && !unicode.IsLetter(r) && !unicode.IsDigit(r) {+ return false+ }+ }+ return true+}++// restAPIHash128 mirrors api.utils.common.hash128 (xxhash128 hex digest).+func restAPIHash128(data string) string {+ return contentFingerprint([]byte(data))+}++// restAPIStableText renders a JSON value as deterministic text: single quotes,+// True/False/None, and matching float formatting. Map keys are sorted to keep+// the fallback document ID stable across runs.+func restAPIStableText(value any) string {+ switch typed := value.(type) {+ case nil:+ return "None"+ case bool:+ if typed {+ return "True"+ }+ return "False"+ case string:+ return restAPIQuoteString(typed)+ case json.Number:+ if strings.ContainsAny(typed.String(), ".eE") {+ if f, err := typed.Float64(); err == nil {+ return restAPIFloatText(f)+ }+ }+ return typed.String()+ case int:+ return strconv.Itoa(typed)+ case int64:+ return strconv.FormatInt(typed, 10)+ case float64:+ return restAPIFloatText(typed)+ case []any:+ parts := make([]string, 0, len(typed))+ for _, item := range typed {+ parts = append(parts, restAPIStableText(item))+ }+ return "[" + strings.Join(parts, ", ") + "]"+ case map[string]any:+ keys := make([]string, 0, len(typed))+ for k := range typed {+ keys = append(keys, k)+ }+ sort.Strings(keys)+ parts := make([]string, 0, len(keys))+ for _, k := range keys {+ parts = append(parts, restAPIQuoteString(k)+": "+restAPIStableText(typed[k]))+ }+ return "{" + strings.Join(parts, ", ") + "}"+ default:+ return fmt.Sprint(value)+ }+}++// restAPIQuoteString renders a string as a single-quoted literal with the+// standard backslash escapes; printable non-ASCII characters are kept as-is.+func restAPIQuoteString(s string) string {+ var b strings.Builder+ b.WriteByte('\'')+ for _, r := range s {+ switch r {+ case '\\':+ b.WriteString(`\\`)+ case '\'':+ b.WriteString(`\'`)+ case '\n':+ b.WriteString(`\n`)+ case '\r':+ b.WriteString(`\r`)+ case '\t':+ b.WriteString(`\t`)+ case '\b':+ b.WriteString(`\b`)+ case '\f':+ b.WriteString(`\f`)+ default:+ if r < 0x20 || r == 0x7f {+ if r < 0x100 {+ b.WriteString(fmt.Sprintf(`\x%02x`, r))+ } else {+ b.WriteString(fmt.Sprintf(`\u%04x`, r))+ }+ } else {+ b.WriteRune(r)+ }+ }+ }+ b.WriteByte('\'')+ return b.String()+}++// restAPIValueText renders the scalar and composite JSON values the connector deals with+// as plain text.+func restAPIValueText(value any) string {+ switch typed := value.(type) {+ case nil:+ return "None"+ case bool:+ if typed {+ return "True"+ }+ return "False"+ case string:+ return typed+ case json.Number:+ if strings.ContainsAny(typed.String(), ".eE") {+ if f, err := typed.Float64(); err == nil {+ return restAPIFloatText(f)+ }+ }+ return typed.String()+ case int:+ return strconv.Itoa(typed)+ case int64:+ return strconv.FormatInt(typed, 10)+ case float64:+ return restAPIFloatText(typed)+ default:+ return restAPIStableText(value)+ }+}++// restAPIFloatText renders floats in a stable textual form.+func restAPIFloatText(f float64) string {+ switch {+ case math.IsInf(f, 1):+ return "inf"+ case math.IsInf(f, -1):+ return "-inf"+ case math.IsNaN(f):+ return "nan"+ case f == math.Trunc(f) && math.Abs(f) < 1e16:+ return strconv.FormatFloat(f, 'f', 1, 64)+ default:+ return strconv.FormatFloat(f, 'g', -1, 64)+ }+}++// coerceRestAPIToText mirrors _coerce_to_text.+func coerceRestAPIToText(value any) string {+ switch typed := value.(type) {+ case nil:+ return ""+ case string:+ return typed+ case json.Number:+ if strings.ContainsAny(typed.String(), ".eE") {+ if f, err := typed.Float64(); err == nil {+ return restAPIFloatText(f)+ }+ }+ return typed.String()+ case int:+ return strconv.Itoa(typed)+ case int64:+ return strconv.FormatInt(typed, 10)+ case float64:+ return restAPIFloatText(typed)+ case bool:+ if typed {+ return "True"+ }+ return "False"+ default:+ return pyJSONDumps(value)+ }+}++// serializeRestAPIMetadataValue mirrors _serialize_metadata_value.+func serializeRestAPIMetadataValue(value any) any {+ switch typed := value.(type) {+ case time.Time:+ return restAPITimeISO(typed)+ case json.Number:+ if f, err := typed.Float64(); err == nil {+ if f == math.Trunc(f) && !math.IsInf(f, 0) && math.Abs(f) < 1e15 {+ return int64(f)+ }+ return f+ }+ return typed.String()+ case int, int64, float64, bool, string:+ return value+ default:+ return pyJSONDumps(value)+ }+}++// pyJSONDumps mirrors json.dumps(value, ensure_ascii=False) with the default+// separators (", " and ": ").+func pyJSONDumps(value any) string {+ var b strings.Builder+ pyJSONDumpsTo(&b, value)+ return b.String()+}++func pyJSONDumpsTo(b *strings.Builder, value any) {+ switch typed := value.(type) {+ case nil:+ b.WriteString("null")+ case bool:+ if typed {+ b.WriteString("true")+ } else {+ b.WriteString("false")+ }+ case string:+ raw, _ := json.Marshal(typed)+ b.Write(raw)+ case json.Number:+ if strings.ContainsAny(typed.String(), ".eE") {+ if f, err := typed.Float64(); err == nil {+ b.WriteString(restAPIFloatText(f))+ return+ }+ }+ b.WriteString(typed.String())+ case int:+ b.WriteString(strconv.Itoa(typed))+ case int64:+ b.WriteString(strconv.FormatInt(typed, 10))+ case float64:+ b.WriteString(restAPIFloatText(typed))+ case []any:+ b.WriteByte('[')+ for i, item := range typed {+ if i > 0 {+ b.WriteString(", ")+ }+ pyJSONDumpsTo(b, item)+ }+ b.WriteByte(']')+ case map[string]any:+ keys := make([]string, 0, len(typed))+ for k := range typed {+ keys = append(keys, k)+ }+ sort.Strings(keys)+ b.WriteByte('{')+ for i, k := range keys {+ if i > 0 {+ b.WriteString(", ")+ }+ raw, _ := json.Marshal(k)+ b.Write(raw)+ b.WriteString(": ")+ pyJSONDumpsTo(b, typed[k])+ }+ b.WriteByte('}')+ default:+ b.WriteString(fmt.Sprint(value))+ }+}++var restAPITagRE = regexp.MustCompile(`<[^>]+>`)+var restAPIWhitespaceRE = regexp.MustCompile(`\s+`)++// restAPIStripHTML mirrors _strip_html.+func restAPIStripHTML(text string) string {+ if !strings.Contains(text, "<") || !strings.Contains(text, ">") {+ return text+ }+ cleaned := restAPITagRE.ReplaceAllString(text, " ")+ return strings.TrimSpace(restAPIWhitespaceRE.ReplaceAllString(cleaned, " "))+}++// restAPITimeISO renders a UTC time as an ISO 8601 string, keeping fractional+// seconds when non-zero.+func restAPITimeISO(t time.Time) string {+ t = t.UTC()+ base := t.Format("2006-01-02T15:04:05")+ if t.Nanosecond() != 0 {+ base += fmt.Sprintf(".%06d", t.Nanosecond()/1000)+ }+ return base + "+00:00"+}++// ---------------------------------------------------------------------------+// Sync session+// ---------------------------------------------------------------------------++// restAPISyncCursor is the page-boundary resume position serialized into+// SyncCheckpoint.Cursor. It stores the next page to fetch for the configured+// pagination type; pagination_type=none never produces a checkpoint.+type restAPISyncCursor struct {+ Page int `json:"page,omitempty"`+ Offset int `json:"offset,omitempty"`+ Cursor string `json:"cursor,omitempty"`+}++// restAPIPageState tracks one fetched page until all of its documents have+// been handed out in emitted batches. nextPos is the resume position after the+// page (nil for pagination_type=none).+type restAPIPageState struct {+ remaining int+ nextPos *restAPISyncCursor+}++// restAPISyncSession streams items lazily so large APIs are not fully+// materialized in memory.+type restAPISyncSession struct {+ connector *RestAPIConnector+ iterator *restAPIItemIterator+ batchSize int+ fromBeginning bool+ windowStart *time.Time+ windowEnd time.Time+ pending []SourceDocument+ pages []restAPIPageState+ drained *restAPISyncCursor+}++// applyResume restores the next page to fetch from a saved checkpoint. Invalid+// or empty cursors are ignored, mirroring the tolerant resume of other+// connectors.+func (s *restAPISyncSession) applyResume(checkpoint *SyncCheckpoint) {+ if checkpoint == nil || checkpoint.Cursor == "" {+ return+ }+ var cursor restAPISyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ return+ }+ switch s.connector.cfg.PaginationType {+ case restAPIPaginationPage:+ if cursor.Page > 0 {+ s.iterator.page = cursor.Page+ }+ case restAPIPaginationOffset:+ if cursor.Offset > 0 {+ s.iterator.offset = cursor.Offset+ }+ case restAPIPaginationCursor:+ if cursor.Cursor != "" {+ s.iterator.cursor = cursor.Cursor+ }+ }+}++// NextBatch returns the next batch of source documents. The batch carries a+// checkpoint only when at least one page has been fully handed out: the+// checkpoint points at the next page to fetch, so a retried task re-fetches at+// most the page where the last batch ended.+func (s *restAPISyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ for len(s.pending) < s.batchSize {+ items, err := s.iterator.nextPage(ctx)+ if err != nil {+ return SyncBatch{}, err+ }+ if items == nil {+ break+ }+ pageDocs := 0+ for _, item := range items {+ doc, err := s.connector.itemToDocument(item)+ if err != nil {+ // Items that fail conversion are skipped with a warning.+ continue+ }+ if !s.fromBeginning && s.windowStart != nil {+ if !restAPIDocInTimeWindow(doc.UpdatedAt, *s.windowStart, s.windowEnd) {+ continue+ }+ }+ s.pending = append(s.pending, doc)+ pageDocs+++ }+ s.pages = append(s.pages, restAPIPageState{remaining: pageDocs, nextPos: s.iterator.positionAfterPage()})+ }+ if len(s.pending) == 0 {+ return SyncBatch{}, io.EOF+ }+ end := s.batchSize+ if end > len(s.pending) {+ end = len(s.pending)+ }+ batch := SyncBatch{Documents: s.pending[:end]}+ s.pending = s.pending[end:]++ // Advance the resume point past every page fully covered by the batches+ // emitted so far. A page whose tail is still pending keeps the checkpoint+ // at the page start, so a retry re-fetches at most that one page.+ for len(s.pages) > 0 {+ if s.pages[0].remaining > end {+ s.pages[0].remaining -= end+ break+ }+ end -= s.pages[0].remaining+ if s.pages[0].nextPos != nil {+ s.drained = s.pages[0].nextPos+ }+ s.pages = s.pages[1:]+ if end == 0 {+ break+ }+ }+ if s.drained != nil {+ if raw, err := json.Marshal(s.drained); err == nil {+ batch.Checkpoint = &SyncCheckpoint{Cursor: string(raw)}+ }+ }+ return batch, nil+}++// Close releases the session.+func (s *restAPISyncSession) Close() error {+ return nil+}++// restAPIDocInTimeWindow mirrors _doc_in_time_window: start <= dt < end.+func restAPIDocInTimeWindow(docUpdatedAt, start, end time.Time) bool {+ dt := docUpdatedAt.UTC()+ return !dt.Before(start) && dt.Before(end)+}diff --git a/internal/syncer/connector/rest_api_test.go b/internal/syncer/connector/rest_api_test.gonew file mode 100644index 0000000000..b0777d8fe4--- /dev/null+++ b/internal/syncer/connector/rest_api_test.go@@ -0,0 +1,1240 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++func withRestAPITestHooks(t *testing.T) {+ t.Helper()+ origLoopback := restAPISSRFAllowLoopback+ origTries := restAPIRetryTries+ origBaseDelay := restAPIRetryBaseDelay+ origMaxDelay := restAPIRetryMaxDelay+ origBackoff := restAPIRetryBackoff+ origJitter := restAPIRetryJitter+ orig429Waits := restAPI429MaxWaits+ orig429Wait := restAPI429DefaultWait+ restAPISSRFAllowLoopback = true+ restAPIRetryTries = 3+ restAPIRetryBaseDelay = time.Millisecond+ restAPIRetryMaxDelay = 10 * time.Millisecond+ restAPIRetryBackoff = 2+ restAPIRetryJitter = 0+ restAPI429MaxWaits = 3+ restAPI429DefaultWait = time.Millisecond+ t.Cleanup(func() {+ restAPISSRFAllowLoopback = origLoopback+ restAPIRetryTries = origTries+ restAPIRetryBaseDelay = origBaseDelay+ restAPIRetryMaxDelay = origMaxDelay+ restAPIRetryBackoff = origBackoff+ restAPIRetryJitter = origJitter+ restAPI429MaxWaits = orig429Waits+ restAPI429DefaultWait = orig429Wait+ })+}++func mustRestAPIConnector(t *testing.T, config map[string]any) *RestAPIConnector {+ t.Helper()+ c, err := NewRestAPIConnector(config)+ if err != nil {+ t.Fatalf("NewRestAPIConnector: %v", err)+ }+ return c+}++func TestNewRestAPIConnectorDefaults(t *testing.T) {+ withRestAPITestHooks(t)+ c := mustRestAPIConnector(t, map[string]any{+ "url": "https://example.com/api",+ "content_fields": "title, body",+ })+ if c.cfg.Method != "GET" {+ t.Fatalf("method=%q want GET", c.cfg.Method)+ }+ if c.cfg.BatchSize != 2 {+ t.Fatalf("batch_size=%d want 2", c.cfg.BatchSize)+ }+ if c.cfg.MaxPages != 1000 {+ t.Fatalf("max_pages=%d want 1000", c.cfg.MaxPages)+ }+ if c.cfg.RequestDelay != 0.5 {+ t.Fatalf("request_delay=%v want 0.5", c.cfg.RequestDelay)+ }+ if len(c.cfg.ContentFields) != 2 || c.cfg.ContentFields[0] != "title" || c.cfg.ContentFields[1] != "body" {+ t.Fatalf("content_fields=%v", c.cfg.ContentFields)+ }+ if c.cfg.AuthType != "none" || c.cfg.PaginationType != "none" {+ t.Fatalf("auth=%q pagination=%q", c.cfg.AuthType, c.cfg.PaginationType)+ }+}++func TestNewRestAPIConnectorClampsNonPositiveBatchSize(t *testing.T) {+ withRestAPITestHooks(t)+ for _, batchSize := range []any{0, -1} {+ c := mustRestAPIConnector(t, map[string]any{+ "url": "https://example.com/api",+ "content_fields": "title",+ "batch_size": batchSize,+ })+ if c.cfg.BatchSize != restAPIDefaultBatchSize {+ t.Fatalf("batch_size=%v parsed to %d, want %d", batchSize, c.cfg.BatchSize, restAPIDefaultBatchSize)+ }+ }+}++type restAPITestReadCloser struct {+ reader io.Reader+ closeErr error+}++func (b *restAPITestReadCloser) Read(p []byte) (int, error) { return b.reader.Read(p) }+func (b *restAPITestReadCloser) Close() error { return b.closeErr }++func TestRestAPICloseIdleBodyPreservesBodyAndCloseError(t *testing.T) {+ closeErr := errors.New("close boom")+ body := &restAPITestReadCloser{reader: strings.NewReader("hello"), closeErr: closeErr}+ wrapped := &restAPICloseIdleBody{body: body, transport: &http.Transport{}}+ got, err := io.ReadAll(wrapped)+ if err != nil || string(got) != "hello" {+ t.Fatalf("read data=%q err=%v", got, err)+ }+ if err := wrapped.Close(); !errors.Is(err, closeErr) {+ t.Fatalf("close err=%v want %v", err, closeErr)+ }+}++func TestNewRestAPIConnectorValidationErrors(t *testing.T) {+ withRestAPITestHooks(t)+ tests := []struct {+ name string+ config map[string]any+ want string+ }{+ {name: "missing url", config: map[string]any{"content_fields": "title"}, want: "Invalid REST API config: url"},+ {name: "unsupported method", config: map[string]any{"url": "https://example.com", "method": "DELETE", "content_fields": "title"}, want: "Unsupported HTTP method 'DELETE'."},+ {name: "unsupported auth", config: map[string]any{"url": "https://example.com", "auth_type": "jwt", "content_fields": "title"}, want: "Unsupported auth_type 'jwt'."},+ {name: "unsupported pagination", config: map[string]any{"url": "https://example.com", "pagination_type": "bad", "content_fields": "title"}, want: "Unsupported pagination_type 'bad'."},+ {name: "missing content fields", config: map[string]any{"url": "https://example.com"}, want: "At least one content field must be configured (content_fields)."},+ {name: "zero max_pages", config: map[string]any{"url": "https://example.com", "max_pages": 0, "content_fields": "title"}, want: "max_pages must be a positive integer"},+ {name: "negative max_pages", config: map[string]any{"url": "https://example.com", "max_pages": -1, "content_fields": "title"}, want: "max_pages must be a positive integer"},+ {name: "bad scheme", config: map[string]any{"url": "ftp://example.com/x", "content_fields": "title"}, want: "Unsupported URL scheme"},+ {name: "localhost", config: map[string]any{"url": "http://localhost/x", "content_fields": "title"}, want: "localhost is blocked"},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ _, err := NewRestAPIConnector(tt.config)+ if err == nil || !strings.Contains(err.Error(), tt.want) {+ t.Fatalf("err=%v want contains %q", err, tt.want)+ }+ var valErr *ConnectorValidationError+ if !errors.As(err, &valErr) {+ t.Fatalf("err=%T want *ConnectorValidationError", err)+ }+ })+ }+}++func TestNewRestAPIConnectorBlocksPrivateAddress(t *testing.T) {+ restAPISSRFAllowLoopback = false+ defer func() { restAPISSRFAllowLoopback = false }()+ _, err := NewRestAPIConnector(map[string]any{+ "url": "http://127.0.0.1:8080/api",+ "content_fields": "title",+ })+ if err == nil || !strings.Contains(err.Error(), "resolves to disallowed address") {+ t.Fatalf("err=%v want private address rejection", err)+ }+}++func TestRestAPITextToDict(t *testing.T) {+ dict := restAPITextToDict(map[string]any{"a": 1, "b": "x"})+ if dict["a"] != "1" || dict["b"] != "x" {+ t.Fatalf("dict=%v", dict)+ }+ jsonDict := restAPITextToDict(`{"a": "1", "b": "x"}`)+ if jsonDict["a"] != "1" || jsonDict["b"] != "x" {+ t.Fatalf("jsonDict=%v", jsonDict)+ }+ lines := restAPITextToDict("# comment\na=1\nb = x y\n")+ if lines["a"] != "1" || lines["b"] != "x y" {+ t.Fatalf("lines=%v", lines)+ }+}++func TestRestAPIAuthPreparation(t *testing.T) {+ withRestAPITestHooks(t)++ apiKey := mustRestAPIConnector(t, map[string]any{+ "url": "https://example.com",+ "content_fields": "title",+ "auth_type": "api_key_header",+ "auth_config": map[string]any{"header_name": "X-Key"},+ "credentials": map[string]any{"api_key": "secret"},+ })+ if err := apiKey.prepare(); err != nil {+ t.Fatalf("prepare api_key: %v", err)+ }+ if got := apiKey.authHeaders["X-Key"]; got != "secret" {+ t.Fatalf("X-Key=%q", got)+ }++ bearer := mustRestAPIConnector(t, map[string]any{+ "url": "https://example.com",+ "content_fields": "title",+ "auth_type": "bearer",+ "credentials": map[string]any{"token": "tok"},+ })+ if err := bearer.prepare(); err != nil {+ t.Fatalf("prepare bearer: %v", err)+ }+ if got := bearer.authHeaders["Authorization"]; got != "Bearer tok" {+ t.Fatalf("Authorization=%q", got)+ }++ basic := mustRestAPIConnector(t, map[string]any{+ "url": "https://example.com",+ "content_fields": "title",+ "auth_type": "basic",+ "credentials": map[string]any{"username": "u", "password": "p"},+ })+ if err := basic.prepare(); err != nil {+ t.Fatalf("prepare basic: %v", err)+ }+ if basic.basicAuth == nil || basic.basicAuth.username != "u" || basic.basicAuth.password != "p" {+ t.Fatalf("basicAuth=%+v", basic.basicAuth)+ }++ missing := mustRestAPIConnector(t, map[string]any{+ "url": "https://example.com",+ "content_fields": "title",+ "auth_type": "bearer",+ })+ err := missing.prepare()+ var credErr *ConnectorMissingCredentialError+ if !errors.As(err, &credErr) || credErr.Message != "REST API (bearer) requires 'token' in credentials" {+ t.Fatalf("err=%v want missing credential error", err)+ }+}++func TestRestAPIFieldExtraction(t *testing.T) {+ withRestAPITestHooks(t)+ c := mustRestAPIConnector(t, map[string]any{+ "url": "https://example.com",+ "content_fields": "title",+ "field_type_hints": map[string]any{"meta.count": "number", "meta.name": "string"},+ "field_default_values": map[string]any{"missing": "fallback"},+ })+ item := map[string]any{+ "title": "T",+ "meta": map[string]any{"count": json.Number("2"), "name": 7},+ "tags": []any{"a", "b"},+ }+ values := extractRestAPIFieldValues(item, "meta.count")+ if len(values) != 1 || values[0] != json.Number("2") {+ t.Fatalf("meta.count=%v", values)+ }+ if got := c.getTypedFieldValue("meta.count", item); got != int64(2) {+ t.Fatalf("typed meta.count=%v (%T)", got, got)+ }+ if got := c.getTypedFieldValue("meta.name", item); got != "7" {+ t.Fatalf("typed meta.name=%v", got)+ }+ tags := extractRestAPIFieldValues(item, "tags[*]")+ if len(tags) != 2 || tags[0] != "a" || tags[1] != "b" {+ t.Fatalf("tags[*]=%v", tags)+ }+ if got := c.getTypedFieldValue("missing", item); got != "fallback" {+ t.Fatalf("default=%v", got)+ }+}++func TestRestAPIParseDatetime(t *testing.T) {+ tests := []struct {+ input any+ want string+ }{+ {input: "2026-08-14T10:00:00Z", want: "2026-08-14T10:00:00Z"},+ {input: "2026-08-14 10:00:00", want: "2026-08-14T10:00:00Z"},+ {input: "2026-08-14T18:00:00+08:00", want: "2026-08-14T10:00:00Z"},+ {input: "2026-08-14", want: "2026-08-14T00:00:00Z"},+ {input: time.Date(2026, 8, 14, 10, 0, 0, 0, time.UTC).Unix(), want: "2026-08-14T10:00:00Z"},+ }+ for _, tt := range tests {+ got := parseRestAPIDatetime(tt.input)+ if got == nil {+ t.Fatalf("parse(%v) = nil", tt.input)+ }+ if got.UTC().Format(time.RFC3339) != tt.want {+ t.Fatalf("parse(%v)=%s want %s", tt.input, got.UTC().Format(time.RFC3339), tt.want)+ }+ }+}++func TestRestAPIItemToDocument(t *testing.T) {+ withRestAPITestHooks(t)+ c := mustRestAPIConnector(t, map[string]any{+ "url": "https://example.com",+ "id_field": "id",+ "content_fields": "title",+ "metadata_fields": "meta.count",+ "poll_timestamp_field": "updated",+ })+ item := map[string]any{+ "id": "abc",+ "title": "Hello <b>World</b>",+ "meta": map[string]any{"count": json.Number("3")},+ "updated": "2026-08-14T10:00:00Z",+ }+ doc, err := c.itemToDocument(item)+ if err != nil {+ t.Fatalf("itemToDocument: %v", err)+ }+ wantID := restAPIHash128("rest_api:" + "abc")+ if doc.SourceID != wantID {+ t.Fatalf("SourceID=%s want %s", doc.SourceID, wantID)+ }+ if doc.SemanticIdentifier != "Hello World" {+ t.Fatalf("sem=%q", doc.SemanticIdentifier)+ }+ if string(doc.Blob) != "Hello World" {+ t.Fatalf("blob=%q", doc.Blob)+ }+ if doc.Extension != ".txt" {+ t.Fatalf("extension=%q", doc.Extension)+ }+ if doc.Metadata["meta.count"] != int64(3) {+ t.Fatalf("metadata=%v", doc.Metadata)+ }+ wantTime := time.Date(2026, 8, 14, 10, 0, 0, 0, time.UTC)+ if !doc.UpdatedAt.Equal(wantTime) {+ t.Fatalf("UpdatedAt=%v want %v", doc.UpdatedAt, wantTime)+ }+ if doc.Fingerprint != contentFingerprint(doc.Blob) {+ t.Fatalf("fingerprint mismatch")+ }+}++func TestRestAPIHash128AndStableText(t *testing.T) {+ if got := restAPIStableText(map[string]any{"a": "x", "b": 1}); got != "{'a': 'x', 'b': 1}" {+ t.Fatalf("stable text dict=%q", got)+ }+ if got := restAPIStableText([]any{"a", 1, true, nil}); got != "['a', 1, True, None]" {+ t.Fatalf("stable text list=%q", got)+ }+ if got := restAPIQuoteString("a'b\n"); got != `'a\'b\n'` {+ t.Fatalf("quoted string=%q", got)+ }+ if got := restAPIValueText(json.Number("1.0")); got != "1.0" {+ t.Fatalf("restAPIValueText float=%q", got)+ }+ if got := restAPIValueText(json.Number("3")); got != "3" {+ t.Fatalf("restAPIValueText int=%q", got)+ }+ if got := restAPIHash128("rest_api:abc"); len(got) != 32 {+ t.Fatalf("hash length=%d want 32", len(got))+ }+}++func TestRestAPIJSONPathAndExtractItems(t *testing.T) {+ response := map[string]any{+ "data": map[string]any{+ "items": []any{+ map[string]any{"id": "1"},+ map[string]any{"id": "2"},+ },+ },+ "paging": map[string]any{"next": "cursor-1"},+ }+ items := restAPIExtractItems(response, "$.data.items")+ if len(items) != 2 {+ t.Fatalf("items=%d want 2", len(items))+ }+ if got := restAPIExtractNextCursor(response, map[string]any{"next_cursor_field": "paging.next"}); got != "" {+ t.Fatalf("next_cursor_field=%q want empty (field is top-level only)", got)+ }+ if got := restAPIExtractNextCursor(response, map[string]any{"next_cursor_path": "$.paging.next"}); got != "cursor-1" {+ t.Fatalf("next_cursor_path=%q", got)+ }+ fallback := restAPIExtractItems(map[string]any{"results": []any{map[string]any{"id": "9"}}}, "")+ if len(fallback) != 1 {+ t.Fatalf("fallback items=%d", len(fallback))+ }+}++func TestRestAPIContentTemplate(t *testing.T) {+ withRestAPITestHooks(t)+ c := mustRestAPIConnector(t, map[string]any{+ "url": "https://example.com",+ "content_fields": "title",+ "metadata_fields": "meta.count",+ "content_template": "Title: {title} / Count: {meta_count}",+ })+ item := map[string]any{"title": "T", "meta": map[string]any{"count": json.Number("3")}}+ if got := c.renderContentTemplate(item); got != "Title: T / Count: 3" {+ t.Fatalf("rendered=%q", got)+ }+ broken := mustRestAPIConnector(t, map[string]any{+ "url": "https://example.com",+ "content_fields": "title",+ "content_template": "{",+ })+ if got := broken.renderContentTemplate(item); got != "T" {+ t.Fatalf("fallback rendered=%q", got)+ }+}++func TestRestAPIFetchPageIntegration(t *testing.T) {+ withRestAPITestHooks(t)++ var requests atomic.Int32+ var mu sync.Mutex+ var gotPath, gotAuth string+ var gotMethod string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ mu.Lock()+ gotPath = r.URL.Path+ gotAuth = r.Header.Get("Authorization")+ gotMethod = r.Method+ mu.Unlock()+ w.Header().Set("Content-Type", "application/json")+ w.Write([]byte(`{"items": [{"id": "1"}]}`))+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL + "/items/{tenant}",+ "query_params": map[string]any{"tenant": "abc"},+ "content_fields": "title",+ "auth_type": "bearer",+ "credentials": map[string]any{"token": "tok"},+ })+ if _, err := c.fetchPage(context.Background(), map[string]any{"page": 1}); err != nil {+ t.Fatalf("fetchPage: %v", err)+ }+ mu.Lock()+ path, auth, method := gotPath, gotAuth, gotMethod+ mu.Unlock()+ if path != "/items/abc" {+ t.Fatalf("path=%q want /items/abc", path)+ }+ if auth != "Bearer tok" {+ t.Fatalf("auth=%q", auth)+ }+ if method != "GET" {+ t.Fatalf("method=%q", method)+ }+ if requests.Load() != 1 {+ t.Fatalf("requests=%d", requests.Load())+ }+}++func TestRestAPIFetchPagePOST(t *testing.T) {+ withRestAPITestHooks(t)+ var mu sync.Mutex+ var gotMethod, gotBody, gotContentType string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ mu.Lock()+ gotMethod = r.Method+ gotContentType = r.Header.Get("Content-Type")+ body, _ := io.ReadAll(r.Body)+ gotBody = string(body)+ mu.Unlock()+ w.Header().Set("Content-Type", "application/json")+ w.Write([]byte(`{}`))+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "method": "POST",+ "content_fields": "title",+ "request_body": map[string]any{"q": "x"},+ })+ if _, err := c.fetchPage(context.Background(), nil); err != nil {+ t.Fatalf("fetchPage: %v", err)+ }+ mu.Lock()+ method, contentType, body := gotMethod, gotContentType, gotBody+ mu.Unlock()+ if method != "POST" {+ t.Fatalf("method=%q", method)+ }+ if contentType != "application/json" {+ t.Fatalf("content-type=%q", contentType)+ }+ if !strings.Contains(body, `"q":"x"`) {+ t.Fatalf("body=%q", body)+ }+}++func TestRestAPIFetchPageErrorMapping(t *testing.T) {+ withRestAPITestHooks(t)+ tests := []struct {+ name string+ status int+ body string+ wantType any+ wantSubstr string+ }{+ {name: "unauthorized", status: 401, body: `{}`, wantType: &ConnectorMissingCredentialError{}, wantSubstr: "REST API authentication failed with status 401"},+ {name: "bad request", status: 400, body: `{}`, wantType: &ConnectorValidationError{}, wantSubstr: "REST API request failed with non-retriable client error status 400"},+ {name: "non json", status: 200, body: `not-json`, wantType: &ConnectorValidationError{}, wantSubstr: "REST API response is not valid JSON"},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(tt.status)+ w.Write([]byte(tt.body))+ }))+ defer server.Close()+ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ })+ _, err := c.fetchPage(context.Background(), nil)+ if err == nil || !strings.Contains(err.Error(), tt.wantSubstr) {+ t.Fatalf("err=%v want contains %q", err, tt.wantSubstr)+ }+ switch tt.wantType.(type) {+ case *ConnectorMissingCredentialError:+ var want *ConnectorMissingCredentialError+ if !errors.As(err, &want) {+ t.Fatalf("err=%T want missing credential error", err)+ }+ case *ConnectorValidationError:+ var want *ConnectorValidationError+ if !errors.As(err, &want) {+ t.Fatalf("err=%T want validation error", err)+ }+ }+ })+ }+}++func TestRestAPIFetchPageRetries(t *testing.T) {+ withRestAPITestHooks(t)+ var attempts atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ n := attempts.Add(1)+ if n < 3 {+ w.WriteHeader(http.StatusInternalServerError)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ w.Write([]byte(`{"ok": true}`))+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ })+ if _, err := c.fetchPage(context.Background(), nil); err != nil {+ t.Fatalf("fetchPage: %v", err)+ }+ if attempts.Load() != 3 {+ t.Fatalf("attempts=%d want 3", attempts.Load())+ }+}++func TestRestAPIFetchPage429RetryAfter(t *testing.T) {+ withRestAPITestHooks(t)+ var attempts atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if attempts.Add(1) == 1 {+ w.Header().Set("Retry-After", "0")+ w.WriteHeader(http.StatusTooManyRequests)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ w.Write([]byte(`{"ok": true}`))+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ })+ if _, err := c.fetchPage(context.Background(), nil); err != nil {+ t.Fatalf("fetchPage: %v", err)+ }+ if attempts.Load() != 2 {+ t.Fatalf("attempts=%d want 2", attempts.Load())+ }+}++func TestRestAPIFetchPageRedirectStripsAuth(t *testing.T) {+ withRestAPITestHooks(t)+ var mu sync.Mutex+ var gotAuth, gotMethod string+ final := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ mu.Lock()+ gotAuth = r.Header.Get("Authorization")+ gotMethod = r.Method+ mu.Unlock()+ w.Header().Set("Content-Type", "application/json")+ w.Write([]byte(`{}`))+ }))+ defer final.Close()+ redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ http.Redirect(w, r, final.URL, http.StatusFound)+ }))+ defer redirector.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": redirector.URL,+ "method": "POST",+ "content_fields": "title",+ "auth_type": "bearer",+ "credentials": map[string]any{"token": "tok"},+ })+ if _, err := c.fetchPage(context.Background(), nil); err != nil {+ t.Fatalf("fetchPage: %v", err)+ }+ mu.Lock()+ auth, method := gotAuth, gotMethod+ mu.Unlock()+ if auth != "" {+ t.Fatalf("cross-origin auth=%q want stripped", auth)+ }+ if method != "GET" {+ t.Fatalf("redirect method=%q want GET", method)+ }+}++func TestRestAPISyncSessionPagination(t *testing.T) {+ withRestAPITestHooks(t)+ var requests atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ page := r.URL.Query().Get("page")+ items := []any{}+ switch page {+ case "", "1":+ items = []any{+ map[string]any{"id": "1", "title": "One", "updated": "2026-08-14T10:00:00Z"},+ map[string]any{"id": "2", "title": "Two", "updated": "2026-08-14T11:00:00Z"},+ }+ case "2":+ items = []any{+ map[string]any{"id": "3", "title": "Three", "updated": "2026-08-14T12:00:00Z"},+ map[string]any{"id": "4", "title": "Four", "updated": "2026-08-14T13:00:00Z"},+ }+ default:+ items = []any{map[string]any{"id": "5", "title": "Five", "updated": "2026-08-14T14:00:00Z"}}+ }+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(map[string]any{"items": items})+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ "id_field": "id",+ "pagination_type": "page",+ "pagination_config": map[string]any{"page_size": 2},+ "batch_size": 2,+ "request_delay": 0,+ })+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()++ var total int+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) > 2 {+ t.Fatalf("batch size=%d", len(batch.Documents))+ }+ total += len(batch.Documents)+ }+ if total != 5 {+ t.Fatalf("total=%d want 5", total)+ }+ if requests.Load() != 3 {+ t.Fatalf("requests=%d want 3", requests.Load())+ }+}++func TestRestAPISyncSessionWindowFilter(t *testing.T) {+ withRestAPITestHooks(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(map[string]any{+ "items": []any{+ map[string]any{"id": "1", "title": "One", "updated": "2026-08-14T10:00:00Z"},+ map[string]any{"id": "2", "title": "Two", "updated": "2026-08-14T12:00:00Z"},+ },+ })+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ "id_field": "id",+ "poll_timestamp_field": "updated",+ })+ start := time.Date(2026, 8, 14, 11, 0, 0, 0, time.UTC)+ end := time.Date(2026, 8, 14, 13, 0, 0, 0, time.UTC)+ session, err := c.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != restAPIHash128("rest_api:2") {+ t.Fatalf("documents=%d want window-filtered id 2", len(batch.Documents))+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("err=%v want EOF", err)+ }+}++func TestRestAPIValidateLive(t *testing.T) {+ withRestAPITestHooks(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ w.Write([]byte(`{"items": []}`))+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ })+ if err := c.ValidateLive(context.Background()); err != nil {+ t.Fatalf("ValidateLive: %v", err)+ }+ if err := c.Validate(context.Background()); err != nil {+ t.Fatalf("Validate: %v", err)+ }+}++func TestRestAPIMaxPages(t *testing.T) {+ withRestAPITestHooks(t)+ var requests atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(map[string]any{+ "items": []any{map[string]any{"id": "1", "title": "One"}, map[string]any{"id": "2", "title": "Two"}},+ })+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ "id_field": "id",+ "pagination_type": "page",+ "pagination_config": map[string]any{"page_size": 2},+ "max_pages": 1,+ "request_delay": 0,+ })+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents=%d want 2", len(batch.Documents))+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("err=%v want EOF", err)+ }+ if requests.Load() != 1 {+ t.Fatalf("requests=%d want 1", requests.Load())+ }+}++func TestRestAPIOpenPruneUnsupported(t *testing.T) {+ withRestAPITestHooks(t)+ c := mustRestAPIConnector(t, map[string]any{+ "url": "https://example.com",+ "content_fields": "title",+ })+ _, err := c.OpenPrune(context.Background(), PruneRequest{})+ if !errors.Is(err, ErrPruneUnsupported) {+ t.Fatalf("err=%v want ErrPruneUnsupported", err)+ }+}++func restAPICheckpointCursor(t *testing.T, batch SyncBatch) restAPISyncCursor {+ t.Helper()+ if batch.Checkpoint == nil || batch.Checkpoint.Cursor == "" {+ t.Fatalf("checkpoint is missing")+ }+ var cursor restAPISyncCursor+ if err := json.Unmarshal([]byte(batch.Checkpoint.Cursor), &cursor); err != nil {+ t.Fatalf("decode checkpoint: %v", err)+ }+ return cursor+}++func TestRestAPIFetchPageServerErrorIsTransient(t *testing.T) {+ withRestAPITestHooks(t)+ var attempts atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ attempts.Add(1)+ w.WriteHeader(http.StatusInternalServerError)+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ })+ _, err := c.fetchPage(context.Background(), nil)+ if err == nil || !strings.Contains(err.Error(), "http 500") {+ t.Fatalf("err=%v want message containing http 500", err)+ }+ if int(attempts.Load()) != restAPIRetryTries {+ t.Fatalf("attempts=%d want %d", attempts.Load(), restAPIRetryTries)+ }+}++func TestRestAPIFetchPage429ExhaustionIsTransient(t *testing.T) {+ withRestAPITestHooks(t)+ var attempts atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ attempts.Add(1)+ w.WriteHeader(http.StatusTooManyRequests)+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ })+ _, err := c.fetchPage(context.Background(), nil)+ if err == nil || !strings.Contains(err.Error(), "too many requests") {+ t.Fatalf("err=%v want message containing too many requests", err)+ }+ var rateErr *RateLimitTriedTooManyTimesError+ if !errors.As(err, &rateErr) {+ t.Fatalf("err=%T want RateLimitTriedTooManyTimesError", err)+ }+ if int(attempts.Load()) != restAPI429MaxWaits {+ t.Fatalf("attempts=%d want %d", attempts.Load(), restAPI429MaxWaits)+ }+}++func TestRestAPISyncSessionNoneNoCheckpoint(t *testing.T) {+ withRestAPITestHooks(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode([]any{+ map[string]any{"id": "1", "title": "One"},+ map[string]any{"id": "2", "title": "Two"},+ map[string]any{"id": "3", "title": "Three"},+ })+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ "id_field": "id",+ "batch_size": 2,+ })+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()++ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if batch.Checkpoint != nil {+ t.Fatalf("pagination none checkpoint=%+v want nil", batch.Checkpoint)+ }+ }+}++func TestRestAPISyncSessionPageResume(t *testing.T) {+ withRestAPITestHooks(t)+ var mu sync.Mutex+ var requested []string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ page := r.URL.Query().Get("page")+ mu.Lock()+ requested = append(requested, page)+ mu.Unlock()+ items := []any{}+ switch page {+ case "1":+ items = []any{+ map[string]any{"id": "1", "title": "One"},+ map[string]any{"id": "2", "title": "Two"},+ }+ case "2":+ items = []any{+ map[string]any{"id": "3", "title": "Three"},+ map[string]any{"id": "4", "title": "Four"},+ }+ default:+ items = []any{map[string]any{"id": "5", "title": "Five"}}+ }+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(map[string]any{"items": items})+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ "id_field": "id",+ "pagination_type": "page",+ "pagination_config": map[string]any{"page_size": 2},+ "batch_size": 2,+ "request_delay": 0,+ })+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ session.Close()+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 ||+ batch.Documents[0].SourceID != restAPIHash128("rest_api:1") ||+ batch.Documents[1].SourceID != restAPIHash128("rest_api:2") {+ t.Fatalf("documents=%v want ids 1,2", batch.Documents)+ }+ cursor := restAPICheckpointCursor(t, batch)+ if cursor.Page != 2 {+ t.Fatalf("cursor=%+v want page 2", cursor)+ }++ resumed, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: batch.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync: %v", err)+ }+ defer resumed.Close()+ var sourceIDs []string+ for {+ b, err := resumed.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("resumed NextBatch: %v", err)+ }+ for _, doc := range b.Documents {+ sourceIDs = append(sourceIDs, doc.SourceID)+ }+ }+ want := []string{restAPIHash128("rest_api:3"), restAPIHash128("rest_api:4"), restAPIHash128("rest_api:5")}+ if strings.Join(sourceIDs, ",") != strings.Join(want, ",") {+ t.Fatalf("sourceIDs=%v want %v", sourceIDs, want)+ }+ mu.Lock()+ got := append([]string(nil), requested...)+ mu.Unlock()+ if len(got) != 3 || got[0] != "1" || got[1] != "2" || got[2] != "3" {+ t.Fatalf("requested pages=%v want [1 2 3]", got)+ }+}++func TestRestAPISyncSessionOffsetResume(t *testing.T) {+ withRestAPITestHooks(t)+ var mu sync.Mutex+ var requested []string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ offset := r.URL.Query().Get("offset")+ mu.Lock()+ requested = append(requested, offset)+ mu.Unlock()+ items := []any{}+ switch offset {+ case "0":+ items = []any{+ map[string]any{"id": "1", "title": "One"},+ map[string]any{"id": "2", "title": "Two"},+ }+ case "2":+ items = []any{+ map[string]any{"id": "3", "title": "Three"},+ map[string]any{"id": "4", "title": "Four"},+ }+ default:+ items = []any{map[string]any{"id": "5", "title": "Five"}}+ }+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(map[string]any{"items": items})+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ "id_field": "id",+ "pagination_type": "offset",+ "pagination_config": map[string]any{"limit": 2},+ "batch_size": 2,+ "request_delay": 0,+ })+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ session.Close()+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 ||+ batch.Documents[0].SourceID != restAPIHash128("rest_api:1") ||+ batch.Documents[1].SourceID != restAPIHash128("rest_api:2") {+ t.Fatalf("documents=%v want ids 1,2", batch.Documents)+ }+ cursor := restAPICheckpointCursor(t, batch)+ if cursor.Offset != 2 {+ t.Fatalf("cursor=%+v want offset 2", cursor)+ }++ resumed, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: batch.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync: %v", err)+ }+ defer resumed.Close()+ var sourceIDs []string+ for {+ b, err := resumed.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("resumed NextBatch: %v", err)+ }+ for _, doc := range b.Documents {+ sourceIDs = append(sourceIDs, doc.SourceID)+ }+ }+ want := []string{restAPIHash128("rest_api:3"), restAPIHash128("rest_api:4"), restAPIHash128("rest_api:5")}+ if strings.Join(sourceIDs, ",") != strings.Join(want, ",") {+ t.Fatalf("sourceIDs=%v want %v", sourceIDs, want)+ }+ mu.Lock()+ got := append([]string(nil), requested...)+ mu.Unlock()+ if len(got) != 3 || got[0] != "0" || got[1] != "2" || got[2] != "4" {+ t.Fatalf("requested offsets=%v want [0 2 4]", got)+ }+}++func TestRestAPISyncSessionCursorResume(t *testing.T) {+ withRestAPITestHooks(t)+ var mu sync.Mutex+ var requested []string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ cursor := r.URL.Query().Get("cursor")+ mu.Lock()+ requested = append(requested, cursor)+ mu.Unlock()+ resp := map[string]any{"items": []any{}}+ switch cursor {+ case "":+ resp["items"] = []any{+ map[string]any{"id": "1", "title": "One"},+ map[string]any{"id": "2", "title": "Two"},+ }+ resp["next_page_token"] = "t2"+ case "t2":+ resp["items"] = []any{+ map[string]any{"id": "3", "title": "Three"},+ map[string]any{"id": "4", "title": "Four"},+ }+ resp["next_page_token"] = "t3"+ default:+ resp["items"] = []any{map[string]any{"id": "5", "title": "Five"}}+ }+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(resp)+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ "id_field": "id",+ "pagination_type": "cursor",+ "pagination_config": map[string]any{"next_cursor_field": "next_page_token"},+ "batch_size": 2,+ "request_delay": 0,+ })+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ session.Close()+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 ||+ batch.Documents[0].SourceID != restAPIHash128("rest_api:1") ||+ batch.Documents[1].SourceID != restAPIHash128("rest_api:2") {+ t.Fatalf("documents=%v want ids 1,2", batch.Documents)+ }+ cursor := restAPICheckpointCursor(t, batch)+ if cursor.Cursor != "t2" {+ t.Fatalf("cursor=%+v want cursor t2", cursor)+ }++ resumed, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: batch.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync: %v", err)+ }+ defer resumed.Close()+ var sourceIDs []string+ for {+ b, err := resumed.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("resumed NextBatch: %v", err)+ }+ for _, doc := range b.Documents {+ sourceIDs = append(sourceIDs, doc.SourceID)+ }+ }+ want := []string{restAPIHash128("rest_api:3"), restAPIHash128("rest_api:4"), restAPIHash128("rest_api:5")}+ if strings.Join(sourceIDs, ",") != strings.Join(want, ",") {+ t.Fatalf("sourceIDs=%v want %v", sourceIDs, want)+ }+ mu.Lock()+ got := append([]string(nil), requested...)+ mu.Unlock()+ if len(got) != 3 || got[0] != "" || got[1] != "t2" || got[2] != "t3" {+ t.Fatalf("requested cursors=%v want [\"\" t2 t3]", got)+ }+}++func TestRestAPISyncSessionCheckpointWaitsForPageBoundary(t *testing.T) {+ withRestAPITestHooks(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ page := r.URL.Query().Get("page")+ items := []any{}+ switch page {+ case "1":+ items = []any{+ map[string]any{"id": "1", "title": "One"},+ map[string]any{"id": "2", "title": "Two"},+ map[string]any{"id": "3", "title": "Three"},+ map[string]any{"id": "4", "title": "Four"},+ }+ default:+ items = []any{map[string]any{"id": "5", "title": "Five"}}+ }+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(map[string]any{"items": items})+ }))+ defer server.Close()++ c := mustRestAPIConnector(t, map[string]any{+ "url": server.URL,+ "content_fields": "title",+ "id_field": "id",+ "pagination_type": "page",+ "pagination_config": map[string]any{"page_size": 4},+ "batch_size": 2,+ "request_delay": 0,+ })+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(first.Documents) != 2 {+ t.Fatalf("documents=%d want 2", len(first.Documents))+ }+ if first.Checkpoint != nil {+ t.Fatalf("mid-page checkpoint=%+v want nil", first.Checkpoint)+ }+ second, err := session.NextBatch(context.Background())+ session.Close()+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(second.Documents) != 2 {+ t.Fatalf("documents=%d want 2", len(second.Documents))+ }+ cursor := restAPICheckpointCursor(t, second)+ if cursor.Page != 2 {+ t.Fatalf("cursor=%+v want page 2", cursor)+ }++ resumed, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: second.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync: %v", err)+ }+ defer resumed.Close()+ b, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resumed NextBatch: %v", err)+ }+ if len(b.Documents) != 1 || b.Documents[0].SourceID != restAPIHash128("rest_api:5") {+ t.Fatalf("resumed documents=%v want id 5", b.Documents)+ }+ if _, err := resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("err=%v want EOF", err)+ }+}diff --git a/internal/syncer/prune_runner.go b/internal/syncer/prune_runner.goindex 8da242c6d3..3cbbeb4db8 100644--- a/internal/syncer/prune_runner.go+++ b/internal/syncer/prune_runner.go@@ -20,8 +20,11 @@ import ("context""errors""io"+ "ragflow/internal/common""ragflow/internal/service"syncerconnector "ragflow/internal/syncer/connector"++ "go.uber.org/zap")// PruneRunner executes one PRUNE task after collecting a full slim snapshot.@@ -47,6 +50,15 @@ func (r *PruneRunner) Run(ctx context.Context, taskContext service.SyncTaskConteKBID: taskContext.Knowledgebase.ID,})if err != nil {+ if errors.Is(err, syncerconnector.ErrPruneUnsupported) {+ // Connectors without a slim snapshot interface (e.g. REST API)+ // complete PRUNE as a no-op without deleting anything.+ if err := r.checkCanceled(ctx, taskContext.Task.ID); err != nil {+ return "", err+ }+ common.Warn("prune unsupported by connector, completing as no-op", zap.String("task_id", taskContext.Task.ID), zap.Error(err))+ return r.taskService.CompletePrune(ctx, taskContext, 0)+ }return "", err}defer session.Close()diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.goindex 4a7be97404..856a88ff87 100644--- a/internal/syncer/syncer.go+++ b/internal/syncer/syncer.go@@ -175,6 +175,7 @@ func registerBuiltInConnectors(registry *syncerconnector.Registry) {registerDAOConnector(registry, "gmail", syncerconnector.NewGmailConnector)registerDAOConnector(registry, "google-drive", syncerconnector.NewGoogleDriveConnector)registerDAOConnector(registry, "google_drive", syncerconnector.NewGoogleDriveConnector)+ registerDAOConnector(registry, "rest_api", syncerconnector.NewRestAPIConnector)registerDAOConnector(registry, "mysql", syncerconnector.NewMySQLConnector)registerDAOConnector(registry, "postgresql", syncerconnector.NewPostgreSQLConnector)}