RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package connector
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"math/rand"
"net"
"net/http"
"net/url"
"regexp"
"sort"
"strconv"
"strings"
"time"
"unicode"
)
// RestAPIConnector is a configuration-driven REST API data source connector.
//
// It mirrors common/data_source/rest_api_connector.py: the same config schema,
// auth derivation, SSRF protection, pagination, JSON extraction and
// item-to-document mapping. The connector is read-only over HTTP(S).
type RestAPIConnector struct {
cfg *restAPIConfig
baseURL string
urlParams map[string]string
credentials map[string]any
authHeaders map[string]string
basicAuth *restAPIBasicAuth
prepared bool
}
// restAPIConfig is the validated connector configuration.
type restAPIConfig struct {
URL string
Method string
Headers map[string]string
QueryParams map[string]string
AuthType string
AuthConfig map[string]any
ItemsPath string
IDField string
ContentFields []string
MetadataFields []string
PaginationType string
PaginationConfig map[string]any
PollTimestampField string
RequestBody map[string]any
FieldTypeHints map[string]string
FieldDefaultValues map[string]any
ContentTemplate string
BatchSize int
MaxPages int
RequestDelay float64
}
// restAPIBasicAuth carries HTTP basic credentials.
type restAPIBasicAuth struct {
username string
password string
}
const (
restAPIDefaultBatchSize = 2 // default per-batch document count
restAPIDefaultMaxPages = 1000 // default page fetch cap
restAPIDefaultRequestDelay = 0.5
restAPIRequestTimeout = 60 * time.Second
restAPIMaxRedirects = 5
restAPIMaxResponseBytes = int64(32 * 1024 * 1024)
)
const (
restAPIAuthNone = "none"
restAPIAuthAPIKeyHeader = "api_key_header"
restAPIAuthBearer = "bearer"
restAPIAuthBasic = "basic"
restAPIPaginationNone = "none"
restAPIPaginationPage = "page"
restAPIPaginationOffset = "offset"
restAPIPaginationCursor = "cursor"
)
// Connector error types let callers (e.g. the connector /test endpoint) map
// failures to stable response codes.
type (
// ConnectorValidationError mirrors ConnectorValidationError.
ConnectorValidationError struct{ Message string }
// ConnectorMissingCredentialError mirrors ConnectorMissingCredentialError.
ConnectorMissingCredentialError struct{ Message string }
// RateLimitTriedTooManyTimesError mirrors RateLimitTriedTooManyTimesError.
RateLimitTriedTooManyTimesError struct{ Message string }
)
func (e *ConnectorValidationError) Error() string { return e.Message }
func (e *ConnectorMissingCredentialError) Error() string { return e.Message }
func (e *RateLimitTriedTooManyTimesError) Error() string { return e.Message }
// ErrPruneUnsupported is returned by OpenPrune for connectors that do not
// support deleted-file pruning; prune tasks then complete without deleting
// anything.
var ErrPruneUnsupported = errors.New("rest API connector does not support prune")
// Retry/backoff knobs for page fetches and rate limiting. They are package
// variables so tests can shrink the delays.
var (
restAPIRetryTries = 5
restAPIRetryBaseDelay = time.Second
restAPIRetryMaxDelay = 30 * time.Second
restAPIRetryBackoff = 2
restAPIRetryJitter = time.Second
restAPI429MaxWaits = 30
restAPI429DefaultWait = 30 * time.Second
)
// NewRestAPIConnector parses a connector config and returns a connector. It
// performs schema validation and the base-URL SSRF check (which resolves the
// hostname to enforce the shared guard) but performs no HTTP request I/O.
// Credentials are read from config["credentials"].
func NewRestAPIConnector(config map[string]any) (*RestAPIConnector, error) {
cfg, err := parseRestAPIConfig(config)
if err != nil {
return nil, err
}
if err := validateRestAPIURLForSSRF(cfg.URL); err != nil {
return nil, err
}
parsed, err := url.Parse(cfg.URL)
if err != nil {
return nil, &ConnectorValidationError{Message: "Invalid REST API config: url"}
}
// Keep the raw path so "{key}" template placeholders survive URL parsing
// (url.URL.String() would percent-encode them).
path := parsed.RawPath
if path == "" {
path = parsed.Path
}
c := &RestAPIConnector{
cfg: cfg,
baseURL: parsed.Scheme + "://" + parsed.Host + path,
urlParams: restAPIURLQueryParams(parsed),
credentials: restAPICredentialsFromConfig(config),
authHeaders: map[string]string{},
}
return c, nil
}
// restAPICredentialsFromConfig extracts and normalizes stored credentials.
func restAPICredentialsFromConfig(config map[string]any) map[string]any {
raw, ok := config["credentials"]
if !ok || raw == nil {
return map[string]any{}
}
if creds, ok := raw.(map[string]any); ok {
return creds
}
if text, ok := raw.(string); ok {
var creds map[string]any
if json.Unmarshal([]byte(text), &creds) == nil && creds != nil {
return creds
}
}
return map[string]any{}
}
// parseRestAPIConfig mirrors RestAPIConnectorConfig + parse_storage_config.
func parseRestAPIConfig(config map[string]any) (*restAPIConfig, error) {
cfg := &restAPIConfig{
Method: "GET",
AuthType: restAPIAuthNone,
PaginationType: restAPIPaginationNone,
BatchSize: restAPIDefaultBatchSize,
MaxPages: restAPIDefaultMaxPages,
RequestDelay: restAPIDefaultRequestDelay,
}
cfg.URL = strings.TrimSpace(stringConfig(config["url"]))
if cfg.URL == "" {
return nil, &ConnectorValidationError{Message: "Invalid REST API config: url\n Field required"}
}
if method := strings.ToUpper(strings.TrimSpace(stringConfig(config["method"]))); method != "" {
cfg.Method = method
}
if cfg.Method != "GET" && cfg.Method != "POST" {
return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported HTTP method '%s'.", cfg.Method)}
}
cfg.Headers = restAPITextToDict(config["headers"])
cfg.QueryParams = restAPITextToDict(config["query_params"])
if auth := strings.TrimSpace(stringConfig(config["auth_type"])); auth != "" {
cfg.AuthType = auth
}
switch cfg.AuthType {
case restAPIAuthNone, restAPIAuthAPIKeyHeader, restAPIAuthBearer, restAPIAuthBasic:
default:
return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported auth_type '%s'.", cfg.AuthType)}
}
cfg.AuthConfig = configAnyMap(config["auth_config"])
cfg.ItemsPath = strings.TrimSpace(stringConfig(config["items_path"]))
cfg.IDField = strings.TrimSpace(stringConfig(config["id_field"]))
cfg.ContentFields = restAPICoerceFieldList(config["content_fields"])
if len(cfg.ContentFields) == 0 {
return nil, &ConnectorValidationError{Message: "At least one content field must be configured (content_fields)."}
}
cfg.MetadataFields = restAPICoerceFieldList(config["metadata_fields"])
if pagination := strings.TrimSpace(stringConfig(config["pagination_type"])); pagination != "" {
cfg.PaginationType = pagination
}
switch cfg.PaginationType {
case restAPIPaginationNone, restAPIPaginationPage, restAPIPaginationOffset, restAPIPaginationCursor:
default:
return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported pagination_type '%s'.", cfg.PaginationType)}
}
cfg.PaginationConfig = configAnyMap(config["pagination_config"])
cfg.PollTimestampField = strings.TrimSpace(stringConfig(config["poll_timestamp_field"]))
cfg.RequestBody = configAnyMap(config["request_body"])
if len(cfg.RequestBody) == 0 {
cfg.RequestBody = configAnyMap(cfg.PaginationConfig["request_body"])
}
cfg.FieldTypeHints = configStringMap(config["field_type_hints"])
cfg.FieldDefaultValues = configAnyMap(config["field_default_values"])
cfg.ContentTemplate = stringConfig(config["content_template"])
if v, ok := restAPIConfigInt(config["batch_size"]); ok {
cfg.BatchSize = v
}
if cfg.BatchSize <= 0 {
cfg.BatchSize = restAPIDefaultBatchSize
}
if v, ok := restAPIConfigInt(config["max_pages"]); ok {
if v <= 0 {
return nil, &ConnectorValidationError{Message: fmt.Sprintf("Invalid REST API config: max_pages must be a positive integer, got %d", v)}
}
cfg.MaxPages = v
}
if v := restAPIConfigFloat(config["request_delay"]); v >= 0 {
cfg.RequestDelay = v
}
return cfg, nil
}
// restAPITextToDict mirrors _text_to_dict: dict, JSON string, or key=value
// lines are all accepted.
func restAPITextToDict(value any) map[string]string {
if value == nil {
return nil
}
if dict, ok := value.(map[string]any); ok {
out := make(map[string]string, len(dict))
for k, v := range dict {
out[k] = fmt.Sprint(v)
}
return out
}
text, ok := value.(string)
if !ok || strings.TrimSpace(text) == "" {
return nil
}
var parsed map[string]any
if json.Unmarshal([]byte(text), &parsed) == nil && parsed != nil {
out := make(map[string]string, len(parsed))
for k, v := range parsed {
out[k] = fmt.Sprint(v)
}
return out
}
out := map[string]string{}
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if idx := strings.Index(line, "="); idx >= 0 {
out[strings.TrimSpace(line[:idx])] = strings.TrimSpace(line[idx+1:])
}
}
return out
}
// restAPICoerceFieldList mirrors the content_fields/metadata_fields coercion.
func restAPICoerceFieldList(value any) []string {
if value == nil {
return nil
}
if text, ok := value.(string); ok {
text = strings.TrimSpace(text)
if text == "" {
return nil
}
parts := strings.Split(text, ",")
out := make([]string, 0, len(parts))
for _, p := range parts {
if p = strings.TrimSpace(p); p != "" {
out = append(out, p)
}
}
return out
}
if list, ok := value.([]any); ok {
out := make([]string, 0, len(list))
for _, item := range list {
if s := strings.TrimSpace(fmt.Sprint(item)); s != "" {
out = append(out, s)
}
}
return out
}
return nil
}
func configAnyMap(value any) map[string]any {
if m, ok := value.(map[string]any); ok {
return m
}
return nil
}
func configStringMap(value any) map[string]string {
if m, ok := value.(map[string]any); ok {
out := make(map[string]string, len(m))
for k, v := range m {
out[k] = stringConfig(v)
}
return out
}
return nil
}
// restAPIConfigInt reads an integer config value without the positive-only
// coercion of configInt so values like max_pages=0 round-trip faithfully.
func restAPIConfigInt(value any) (int, bool) {
switch typed := value.(type) {
case int:
return typed, true
case int64:
return int(typed), true
case float64:
return int(typed), true
case json.Number:
if v, err := typed.Int64(); err == nil {
return int(v), true
}
case string:
if v, err := strconv.Atoi(strings.TrimSpace(typed)); err == nil {
return v, true
}
}
return 0, false
}
func restAPIConfigFloat(value any) float64 {
switch typed := value.(type) {
case float64:
return typed
case int:
return float64(typed)
case int64:
return float64(typed)
case json.Number:
if v, err := typed.Float64(); err == nil {
return v
}
case string:
if v, err := strconv.ParseFloat(strings.TrimSpace(typed), 64); err == nil {
return v
}
}
return -1
}
// restAPIURLQueryParams splits the URL query string into a string map, keeping
// the last value per key.
func restAPIURLQueryParams(parsed *url.URL) map[string]string {
if parsed.RawQuery == "" {
return nil
}
values := parsed.Query()
out := make(map[string]string, len(values))
for k, list := range values {
if len(list) > 0 {
out[k] = list[len(list)-1]
}
}
return out
}
// prepare builds auth headers from credentials, mirroring _build_auth.
func (c *RestAPIConnector) prepare() error {
if c.prepared {
return nil
}
c.authHeaders = map[string]string{}
c.basicAuth = nil
switch c.cfg.AuthType {
case restAPIAuthNone:
case restAPIAuthAPIKeyHeader:
headerName := strings.TrimSpace(stringConfig(c.cfg.AuthConfig["header_name"]))
apiKey := stringConfig(c.credentials["api_key"])
if apiKey == "" {
apiKey = stringConfig(c.cfg.AuthConfig["api_key_value"])
}
if apiKey == "" {
apiKey = stringConfig(c.cfg.AuthConfig["api_key"])
}
if headerName == "" || apiKey == "" {
return &ConnectorMissingCredentialError{Message: "REST API (api_key_header) requires 'header_name' in auth_config and 'api_key' in credentials"}
}
c.authHeaders[headerName] = apiKey
case restAPIAuthBearer:
token := stringConfig(c.credentials["token"])
if token == "" {
token = stringConfig(c.cfg.AuthConfig["token"])
}
if token == "" {
return &ConnectorMissingCredentialError{Message: "REST API (bearer) requires 'token' in credentials"}
}
c.authHeaders["Authorization"] = "Bearer " + token
case restAPIAuthBasic:
username := stringConfig(c.credentials["username"])
if username == "" {
username = stringConfig(c.cfg.AuthConfig["username"])
}
password := stringConfig(c.credentials["password"])
if password == "" {
password = stringConfig(c.cfg.AuthConfig["password"])
}
if username == "" || password == "" {
return &ConnectorMissingCredentialError{Message: "REST API (basic) requires 'username' and 'password'"}
}
c.basicAuth = &restAPIBasicAuth{username: username, password: password}
default:
return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported auth_type: %s", c.cfg.AuthType)}
}
c.prepared = true
return nil
}
// Validate validates configuration and credential presence without network
// I/O; live connectivity validation is performed by ValidateLive.
func (c *RestAPIConnector) Validate(ctx context.Context) error {
if c == nil {
return &ConnectorValidationError{Message: "rest API connector is nil"}
}
return c.prepare()
}
// ValidateLive mirrors RestAPIConnector.validate_config: schema validation plus
// a live single-page fetch with the stored credentials.
func (c *RestAPIConnector) ValidateLive(ctx context.Context) error {
if c == nil {
return &ConnectorValidationError{Message: "rest API connector is nil"}
}
if err := c.prepare(); err != nil {
return err
}
params := map[string]any{}
switch c.cfg.PaginationType {
case restAPIPaginationPage:
page := 1
if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["start_page"]); ok {
page = v
}
perPage, err := c.resolvePageSize()
if err != nil {
return err
}
c.applyPagePagination(params, page, perPage)
case restAPIPaginationOffset:
perPage, err := c.resolvePageSize()
if err != nil {
return err
}
offset := 0
if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["start_offset"]); ok {
offset = v
}
limit := perPage
if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["limit"]); ok {
limit = v
}
if limit <= 0 {
limit = perPage
}
c.applyOffsetPagination(params, offset, limit)
case restAPIPaginationCursor:
if cursor := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["initial_cursor"])); cursor != "" {
c.applyCursorPagination(params, cursor)
}
}
_, err := c.fetchPage(ctx, params)
return err
}
// ValidateConnectorSetting validates REST API settings from an unsaved config.
func (c *RestAPIConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
defer cancel()
return c.ValidateLive(ctx)
}
// OpenSync opens one sync session. Incremental runs filter documents to the
// request window (start <= updated_at < end).
func (c *RestAPIConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
if err := c.prepare(); err != nil {
return nil, err
}
var windowStart *time.Time
if !request.FromBeginning && request.WindowStart != nil {
ws := request.WindowStart.UTC()
windowStart = &ws
}
session := &restAPISyncSession{
connector: c,
iterator: newRestAPIItemIterator(c),
batchSize: c.cfg.BatchSize,
fromBeginning: request.FromBeginning,
windowStart: windowStart,
windowEnd: request.WindowEnd.UTC(),
}
if err := session.applyResume(request.Resume); err != nil {
return nil, err
}
return session, nil
}
// OpenPrune reports prune as unsupported for sources that cannot enumerate a
// complete slim snapshot; prune tasks then complete as no-ops.
func (c *RestAPIConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
return nil, ErrPruneUnsupported
}
// ---------------------------------------------------------------------------
// HTTP fetching
// ---------------------------------------------------------------------------
// fetchPage fetches a single page with retry, mirroring _fetch_page.
func (c *RestAPIConnector) fetchPage(ctx context.Context, params map[string]any) (any, error) {
delay := restAPIRetryBaseDelay
var lastErr error
for attempt := 0; attempt < restAPIRetryTries; attempt++ {
response, err := c.fetchPageOnce(ctx, params)
if err == nil {
return response, nil
}
if ctx.Err() != nil {
return nil, ctx.Err()
}
var (
valErr *ConnectorValidationError
credErr *ConnectorMissingCredentialError
rateErr *RateLimitTriedTooManyTimesError
)
if errors.As(err, &valErr) || errors.As(err, &credErr) || errors.As(err, &rateErr) {
return nil, err
}
lastErr = err
if attempt == restAPIRetryTries-1 {
break
}
sleepFor := delay + time.Duration(rand.Float64()*float64(restAPIRetryJitter))
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(sleepFor):
}
delay = time.Duration(float64(delay) * float64(restAPIRetryBackoff))
if delay > restAPIRetryMaxDelay {
delay = restAPIRetryMaxDelay
}
}
return nil, lastErr
}
// fetchPageOnce performs one page fetch including the 429 wait loop and manual
// redirect handling with per-hop SSRF validation and DNS pinning.
func (c *RestAPIConnector) fetchPageOnce(ctx context.Context, params map[string]any) (any, error) {
if err := c.prepare(); err != nil {
return nil, err
}
merged := map[string]any{}
for k, v := range c.urlParams {
merged[k] = v
}
for k, v := range c.cfg.QueryParams {
merged[k] = v
}
for k, v := range params {
merged[k] = v
}
currentURL, queryParams := c.buildURLWithTemplates(merged)
currentMethod := c.cfg.Method
currentHeaders := make(map[string]string, len(c.cfg.Headers)+len(c.authHeaders))
for k, v := range c.cfg.Headers {
currentHeaders[k] = v
}
for k, v := range c.authHeaders {
currentHeaders[k] = v
}
currentBody := c.cfg.RequestBody
var bodyBytes []byte
if currentMethod == http.MethodPost {
if currentBody == nil {
bodyBytes = []byte("{}")
} else {
var err error
bodyBytes, err = json.Marshal(currentBody)
if err != nil {
return nil, &ConnectorValidationError{Message: "REST API request body is not valid JSON"}
}
}
}
currentAuth := c.basicAuth
previousOrigin := connectorOrigin(currentURL)
sensitiveHeaders := make([]string, 0, len(c.authHeaders))
for name := range c.authHeaders {
sensitiveHeaders = append(sensitiveHeaders, name)
}
resp, err := connectorRequest(ctx, connectorRequestOptions{
Method: currentMethod,
RawURL: currentURL,
Body: bodyBytes,
Headers: currentHeaders,
SensitiveHeaders: sensitiveHeaders,
Timeout: restAPIRequestTimeout,
MaxRedirects: restAPIMaxRedirects,
Prepare: func(req *http.Request, hop connectorRequestHop) error {
if hop.Method == http.MethodPost && req.Header.Get("Content-Type") == "" {
req.Header.Set("Content-Type", "application/json")
}
if currentAuth != nil {
req.SetBasicAuth(currentAuth.username, currentAuth.password)
}
if encoded := restAPIEncodeParams(queryParams); encoded != "" {
sep := "?"
if strings.Contains(hop.URL, "?") {
sep = "&"
}
fullURL, err := url.Parse(hop.URL + sep + encoded)
if err != nil {
return err
}
req.URL = fullURL
}
return nil
},
RetryStatus: func(resp *http.Response, attempt int) (time.Duration, bool) {
if resp.StatusCode != http.StatusTooManyRequests {
return 0, false
}
if attempt >= restAPI429MaxWaits {
return 0, false
}
wait := restAPI429DefaultWait
if raw := resp.Header.Get("Retry-After"); raw != "" {
if seconds, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && seconds >= 0 {
wait = time.Duration(seconds) * time.Second
}
}
return wait, true
},
NextHop: func(nextURL string, status int, hop connectorRequestHop) connectorRequestHop {
nextOrigin := connectorOrigin(nextURL)
if nextOrigin != "" && nextOrigin != previousOrigin {
currentAuth = nil
}
previousOrigin = nextOrigin
if status == http.StatusMovedPermanently || status == http.StatusFound || status == http.StatusSeeOther {
hop.Method = http.MethodGet
hop.Body = nil
queryParams = nil
}
hop.URL = nextURL
return hop
},
})
if err != nil {
var unsafe *connectorUnsafeURLError
if errors.As(err, &unsafe) {
return nil, &ConnectorValidationError{Message: "Unsafe REST API URL: " + unsafe.Err.Error()}
}
return nil, err
}
if resp.StatusCode == http.StatusTooManyRequests {
resp.Body.Close()
return nil, &RateLimitTriedTooManyTimesError{Message: fmt.Sprintf("REST API rate limited: exceeded '%d' retries (too many requests)", restAPI429MaxWaits)}
}
return c.handleResponse(resp)
}
func (c *RestAPIConnector) handleResponse(resp *http.Response) (any, error) {
defer resp.Body.Close()
status := resp.StatusCode
switch {
case status == http.StatusUnauthorized || status == http.StatusForbidden:
return nil, &ConnectorMissingCredentialError{Message: fmt.Sprintf("REST API authentication failed with status %d", status)}
case status >= 400 && status < 500 && status != http.StatusTooManyRequests:
return nil, &ConnectorValidationError{Message: fmt.Sprintf("REST API request failed with non-retriable client error status %d", status)}
case status >= 500:
// Keep the "http <status>" wording so the failure surfaces clearly in
// the sync task error message.
return nil, fmt.Errorf("REST API request failed with http %d", status)
}
raw, err := io.ReadAll(io.LimitReader(resp.Body, restAPIMaxResponseBytes+1))
if err != nil {
return nil, err
}
if int64(len(raw)) > restAPIMaxResponseBytes {
return nil, &ConnectorValidationError{Message: fmt.Sprintf("REST API response exceeds the %d byte limit", restAPIMaxResponseBytes)}
}
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.UseNumber()
var value any
if err := decoder.Decode(&value); err != nil {
return nil, &ConnectorValidationError{Message: "REST API response is not valid JSON"}
}
return value, nil
}
// buildURLWithTemplates substitutes {key} placeholders in the URL and returns
// the remaining query parameters.
func (c *RestAPIConnector) buildURLWithTemplates(params map[string]any) (string, map[string]any) {
u := c.baseURL
remaining := make(map[string]any, len(params))
for k, v := range params {
remaining[k] = v
}
for k, v := range params {
placeholder := "{" + k + "}"
if strings.Contains(u, placeholder) {
u = strings.ReplaceAll(u, placeholder, restAPIParamString(v))
delete(remaining, k)
}
}
return u, remaining
}
func restAPIParamString(value any) string {
switch typed := value.(type) {
case string:
return typed
case json.Number:
return typed.String()
case bool:
if typed {
return "True"
}
return "False"
case nil:
return ""
default:
return fmt.Sprint(value)
}
}
func restAPIEncodeParams(params map[string]any) string {
if len(params) == 0 {
return ""
}
values := url.Values{}
for k, v := range params {
values.Set(k, restAPIParamString(v))
}
return values.Encode()
}
// ---------------------------------------------------------------------------
// SSRF protection
// ---------------------------------------------------------------------------
// validateRestAPIURLForSSRF is the config-time SSRF check for the REST API
// connector. It delegates to the shared connector guard, which validates the
// scheme, rejects localhost and non-public literal addresses, and resolves the
// hostname. The per-request path re-validates and DNS-pins every hop.
func validateRestAPIURLForSSRF(rawURL string) error {
return validateConnectorURL(rawURL)
}
// assertRestAPIURLSafe mirrors ssrf_guard.assert_url_is_safe: every resolved
// address must be globally routable. It returns the hostname and the first
// validated IP so the caller can pin DNS.
func assertRestAPIURLSafe(_ context.Context, rawURL string) (string, net.IP, error) {
return assertConnectorURLSafe(rawURL)
}
// ---------------------------------------------------------------------------
// Pagination
// ---------------------------------------------------------------------------
// resolvePageSize mirrors _resolve_page_size.
func (c *RestAPIConnector) resolvePageSize() (int, error) {
if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["page_size"]); ok && v > 0 {
return v, nil
}
sizeParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["page_size_param"]))
if sizeParam == "" {
sizeParam = strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["limit_param"]))
}
if sizeParam != "" {
for _, source := range []map[string]string{c.cfg.QueryParams, c.urlParams} {
if raw, ok := source[sizeParam]; ok {
if v, err := strconv.Atoi(raw); err == nil && v > 0 {
return v, nil
}
}
}
}
return c.cfg.BatchSize, nil
}
func (c *RestAPIConnector) applyPagePagination(params map[string]any, page, perPage int) {
pageParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["page_param"]))
if pageParam == "" {
pageParam = "page"
}
params[pageParam] = page
if sizeParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["page_size_param"])); sizeParam != "" {
params[sizeParam] = perPage
}
}
func (c *RestAPIConnector) applyOffsetPagination(params map[string]any, offset, limit int) {
offsetParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["offset_param"]))
if offsetParam == "" {
offsetParam = "offset"
}
params[offsetParam] = offset
if limitParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["limit_param"])); limitParam != "" {
params[limitParam] = limit
}
}
func (c *RestAPIConnector) applyCursorPagination(params map[string]any, cursor string) {
cursorParam := strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["cursor_param"]))
if cursorParam == "" {
cursorParam = "cursor"
}
params[cursorParam] = cursor
}
// restAPIItemIterator mirrors _iter_items: it walks pages applying the
// configured pagination and stopping when the source reports no more items.
type restAPIItemIterator struct {
c *RestAPIConnector
pageCount int
page int
offset int
limit int
perPage int
cursor string
finished bool
lastPos *restAPISyncCursor
seenCursors map[string]struct{}
}
func newRestAPIItemIterator(c *RestAPIConnector) *restAPIItemIterator {
it := &restAPIItemIterator{c: c, seenCursors: map[string]struct{}{}}
if perPage, err := c.resolvePageSize(); err == nil {
it.perPage = perPage
} else {
it.perPage = c.cfg.BatchSize
}
if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["start_page"]); ok {
it.page = v
} else {
it.page = 1
}
if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["start_offset"]); ok {
it.offset = v
}
it.limit = it.perPage
if v, ok := restAPIConfigInt(c.cfg.PaginationConfig["limit"]); ok {
it.limit = v
}
if it.limit <= 0 {
it.limit = it.perPage
}
it.cursor = strings.TrimSpace(stringConfig(c.cfg.PaginationConfig["initial_cursor"]))
if it.cursor != "" {
it.seenCursors[it.cursor] = struct{}{}
}
return it
}
// nextPage returns the items of the next page, or nil when iteration is done.
func (it *restAPIItemIterator) nextPage(ctx context.Context) ([]map[string]any, error) {
if it.finished || it.pageCount >= it.c.cfg.MaxPages {
return nil, nil
}
it.lastPos = it.positionCursor()
params := map[string]any{}
switch it.c.cfg.PaginationType {
case restAPIPaginationPage:
it.c.applyPagePagination(params, it.page, it.perPage)
case restAPIPaginationOffset:
it.c.applyOffsetPagination(params, it.offset, it.limit)
case restAPIPaginationCursor:
if it.cursor != "" {
it.c.applyCursorPagination(params, it.cursor)
}
}
if it.pageCount > 0 && it.c.cfg.RequestDelay > 0 {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(time.Duration(it.c.cfg.RequestDelay * float64(time.Second))):
}
}
response, err := it.c.fetchPage(ctx, params)
if err != nil {
var (
valErr *ConnectorValidationError
credErr *ConnectorMissingCredentialError
)
if errors.As(err, &valErr) || errors.As(err, &credErr) {
return nil, err
}
return nil, &ConnectorValidationError{Message: fmt.Sprintf("REST API page fetch failed: %v", err)}
}
items := restAPIExtractItems(response, it.c.cfg.ItemsPath)
hasNext, reportsHasNext := restAPIExtractHasNextPage(response, it.c.cfg.PaginationConfig)
if len(items) == 0 && !(it.c.cfg.PaginationType == restAPIPaginationCursor && reportsHasNext && hasNext) {
it.finished = true
return nil, nil
}
it.pageCount++
switch it.c.cfg.PaginationType {
case restAPIPaginationNone:
it.finished = true
case restAPIPaginationPage:
if len(items) < it.perPage {
it.finished = true
} else {
it.page++
}
case restAPIPaginationOffset:
if len(items) < it.limit {
it.finished = true
} else {
it.offset += it.limit
}
case restAPIPaginationCursor:
if reportsHasNext && !hasNext {
it.finished = true
break
}
next := restAPIExtractNextCursor(response, it.c.cfg.PaginationConfig)
if next == "" {
it.finished = true
} else if _, repeated := it.seenCursors[next]; repeated {
return nil, fmt.Errorf("rest api pagination repeated cursor %q: %w", next, ErrSyncResumeInvalid)
} else {
it.seenCursors[next] = struct{}{}
it.cursor = next
}
}
return items, nil
}
// positionCursor returns the position of the page that was just fetched. It is
// nil for pagination_type=none, which never emits checkpoints.
func (it *restAPIItemIterator) positionCursor() *restAPISyncCursor {
switch it.c.cfg.PaginationType {
case restAPIPaginationPage:
return &restAPISyncCursor{Page: it.page, SourceID: ""}
case restAPIPaginationOffset:
return &restAPISyncCursor{Offset: it.offset, SourceID: ""}
case restAPIPaginationCursor:
return &restAPISyncCursor{Cursor: it.cursor, SourceID: ""}
}
return nil
}
// currentPagePosition returns the position of the page fetched by the most
// recent nextPage call. It keeps the page position, not the next page.
func (it *restAPIItemIterator) currentPagePosition() *restAPISyncCursor {
if it.lastPos == nil {
return nil
}
pos := *it.lastPos
return &pos
}
// ---------------------------------------------------------------------------
// JSON extraction
// ---------------------------------------------------------------------------
// restAPIExtractItems mirrors _extract_items.
func restAPIExtractItems(response any, itemsPath string) []map[string]any {
var items []any
if itemsPath != "" {
matches, err := restAPIJSONPathMatches(response, itemsPath)
if err != nil {
return nil
}
if len(matches) == 0 {
return nil
}
if len(matches) == 1 {
if list, ok := matches[0].([]any); ok {
items = list
} else {
items = matches
}
} else {
items = matches
}
} else if list, ok := response.([]any); ok {
items = list
} else if dict, ok := response.(map[string]any); ok {
for _, key := range []string{"items", "results", "data", "records"} {
if value, ok := dict[key]; ok {
if list, ok := value.([]any); ok {
items = list
break
}
}
}
if items == nil {
for _, value := range dict {
if list, ok := value.([]any); ok {
items = list
break
}
}
}
}
out := make([]map[string]any, 0, len(items))
for _, item := range items {
if mapping, ok := item.(map[string]any); ok {
out = append(out, mapping)
}
}
return out
}
func restAPIExtractHasNextPage(response any, paginationConfig map[string]any) (bool, bool) {
field := strings.TrimSpace(stringConfig(paginationConfig["has_next_page_field"]))
if field == "" {
return false, false
}
dict, ok := response.(map[string]any)
if !ok {
return false, false
}
value, ok := dict[field].(bool)
return value, ok
}
// restAPIExtractNextCursor mirrors _extract_next_cursor.
func restAPIExtractNextCursor(response any, paginationConfig map[string]any) string {
cursorPath := strings.TrimSpace(stringConfig(paginationConfig["next_cursor_path"]))
if cursorPath == "" {
field := strings.TrimSpace(stringConfig(paginationConfig["next_cursor_field"]))
if field != "" {
if dict, ok := response.(map[string]any); ok {
if value, ok := dict[field]; ok && value != nil {
return fmt.Sprint(value)
}
}
}
return ""
}
matches, err := restAPIJSONPathMatches(response, cursorPath)
if err != nil || len(matches) == 0 || matches[0] == nil {
return ""
}
return fmt.Sprint(matches[0])
}
const (
restAPIJSONPathField = iota
restAPIJSONPathIndex
restAPIJSONPathWildcard
)
type restAPIJSONPathToken struct {
kind int
key string
index int
recursive bool
}
// restAPIJSONPathMatches implements the commonly used subset of jsonpath: $,
// dot keys, [n], [*], and .. recursive descent.
func restAPIJSONPathMatches(root any, path string) ([]any, error) {
tokens, err := parseRestAPIJSONPath(path)
if err != nil {
return nil, err
}
current := []any{root}
for _, token := range tokens {
next := []any{}
for _, value := range current {
if token.recursive {
restAPIJSONPathCollectRecursive(value, token, &next)
continue
}
switch token.kind {
case restAPIJSONPathField:
if dict, ok := value.(map[string]any); ok {
if child, ok := dict[token.key]; ok {
next = append(next, child)
}
}
case restAPIJSONPathIndex:
if list, ok := value.([]any); ok && token.index >= 0 && token.index < len(list) {
next = append(next, list[token.index])
}
case restAPIJSONPathWildcard:
switch typed := value.(type) {
case []any:
next = append(next, typed...)
case map[string]any:
for _, child := range typed {
next = append(next, child)
}
}
}
}
current = next
if len(current) == 0 {
break
}
}
return current, nil
}
func restAPIJSONPathCollectRecursive(value any, token restAPIJSONPathToken, out *[]any) {
if token.kind != restAPIJSONPathField {
return
}
if dict, ok := value.(map[string]any); ok {
if child, ok := dict[token.key]; ok {
*out = append(*out, child)
}
for _, nested := range dict {
restAPIJSONPathCollectRecursive(nested, token, out)
}
}
if list, ok := value.([]any); ok {
for _, nested := range list {
restAPIJSONPathCollectRecursive(nested, token, out)
}
}
}
func parseRestAPIJSONPath(path string) ([]restAPIJSONPathToken, error) {
s := strings.TrimSpace(path)
if strings.HasPrefix(s, "$") {
s = s[1:]
}
var tokens []restAPIJSONPathToken
pendingRecursive := false
i := 0
for i < len(s) {
switch s[i] {
case '.':
if i+1 < len(s) && s[i+1] == '.' {
pendingRecursive = true
i += 2
} else {
i++
}
case '[':
end := strings.IndexByte(s[i:], ']')
if end < 0 {
return nil, fmt.Errorf("unterminated bracket in JSONPath")
}
content := strings.TrimSpace(s[i+1 : i+end])
token := restAPIJSONPathToken{recursive: pendingRecursive}
pendingRecursive = false
switch {
case content == "*":
token.kind = restAPIJSONPathWildcard
default:
if n, err := strconv.Atoi(content); err == nil {
token.kind = restAPIJSONPathIndex
token.index = n
} else {
token.kind = restAPIJSONPathField
token.key = strings.Trim(content, "'\"")
}
}
tokens = append(tokens, token)
i += end + 1
default:
start := i
for i < len(s) && s[i] != '.' && s[i] != '[' {
i++
}
key := s[start:i]
if key == "" {
return nil, fmt.Errorf("empty JSONPath segment")
}
tokens = append(tokens, restAPIJSONPathToken{kind: restAPIJSONPathField, key: key, recursive: pendingRecursive})
pendingRecursive = false
}
}
return tokens, nil
}
// ---------------------------------------------------------------------------
// Field extraction and item-to-document mapping
// ---------------------------------------------------------------------------
var restAPIFieldSegmentRE = regexp.MustCompile(`^([^\[\]]+)(\[(\d+|\*)\])?$`)
// extractRestAPIFieldValues mirrors _extract_field_values: dot-notation with
// optional [index] / [*] segments.
func extractRestAPIFieldValues(item map[string]any, path string) []any {
if path == "" {
return nil
}
current := []any{item}
for _, segment := range strings.Split(path, ".") {
if segment == "" {
return nil
}
match := restAPIFieldSegmentRE.FindStringSubmatch(segment)
key := segment
index := ""
if match != nil {
key = match[1]
index = match[3]
}
next := []any{}
for _, value := range current {
dict, ok := value.(map[string]any)
if !ok {
continue
}
child, ok := dict[key]
if !ok || child == nil {
continue
}
if index == "" {
next = append(next, child)
continue
}
list, ok := child.([]any)
if !ok {
continue
}
if index == "*" {
next = append(next, list...)
continue
}
idx, err := strconv.Atoi(index)
if err != nil {
continue
}
if idx >= 0 && idx < len(list) {
next = append(next, list[idx])
}
}
current = next
if len(current) == 0 {
break
}
}
return current
}
func extractRestAPIField(item map[string]any, path string) any {
values := extractRestAPIFieldValues(item, path)
if len(values) == 0 {
return nil
}
if len(values) == 1 {
return values[0]
}
return values
}
// getTypedFieldValue mirrors _get_typed_field_value.
func (c *RestAPIConnector) getTypedFieldValue(path string, item map[string]any) any {
values := extractRestAPIFieldValues(item, path)
if len(values) == 0 {
return c.cfg.FieldDefaultValues[path]
}
hint := c.cfg.FieldTypeHints[path]
converted := make([]any, 0, len(values))
for _, value := range values {
if cv := convertRestAPITypedValue(value, hint); cv != nil {
converted = append(converted, cv)
}
}
if len(converted) == 0 {
return nil
}
if len(converted) == 1 {
return converted[0]
}
parts := make([]string, 0, len(converted))
for _, value := range converted {
parts = append(parts, coerceRestAPIToText(value))
}
return strings.Join(parts, ", ")
}
func convertRestAPITypedValue(value any, hint string) any {
switch hint {
case "string":
return restAPIValueText(value)
case "number":
f, ok := restAPIToFloat(value)
if !ok {
return nil
}
if f == math.Trunc(f) && !math.IsInf(f, 0) && math.Abs(f) < 1e15 {
return int64(f)
}
return f
case "date":
if t, ok := value.(time.Time); ok {
return restAPITimeISO(t)
}
if t := parseRestAPIDatetime(value); t != nil {
return restAPITimeISO(*t)
}
return restAPIValueText(value)
default:
return value
}
}
func restAPIToFloat(value any) (float64, bool) {
switch typed := value.(type) {
case json.Number:
f, err := typed.Float64()
return f, err == nil
case float64:
return typed, true
case int:
return float64(typed), true
case int64:
return float64(typed), true
case string:
f, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
return f, err == nil
default:
return 0, false
}
}
// extractTimestamp mirrors _extract_timestamp.
func (c *RestAPIConnector) extractTimestamp(item map[string]any) *time.Time {
if c.cfg.PollTimestampField == "" {
return nil
}
value := extractRestAPIField(item, c.cfg.PollTimestampField)
if list, ok := value.([]any); ok && len(list) > 0 {
value = list[0]
}
return parseRestAPIDatetime(value)
}
// parseRestAPIDatetime mirrors _parse_datetime.
func parseRestAPIDatetime(value any) *time.Time {
if value == nil {
return nil
}
switch typed := value.(type) {
case time.Time:
t := typed.UTC()
return &t
case json.Number:
if f, err := typed.Float64(); err == nil {
return restAPITimestampFromFloat(f)
}
case float64:
return restAPITimestampFromFloat(typed)
case int:
return restAPITimestampFromFloat(float64(typed))
case int64:
return restAPITimestampFromFloat(float64(typed))
case string:
text := strings.TrimSpace(typed)
if text == "" {
return nil
}
layouts := []string{
"2006-01-02T15:04:05.999999999Z",
"2006-01-02T15:04:05Z",
"2006-01-02 15:04:05",
"2006-01-02",
}
for _, layout := range layouts {
if t, err := time.Parse(layout, text); err == nil {
t = t.UTC()
return &t
}
}
normalized := strings.ReplaceAll(text, " ", "T")
if strings.HasSuffix(normalized, "Z") {
normalized = normalized[:len(normalized)-1] + "+00:00"
}
for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05.999999999-07:00", "2006-01-02T15:04:05-07:00"} {
if t, err := time.Parse(layout, normalized); err == nil {
t = t.UTC()
return &t
}
}
}
return nil
}
func restAPITimestampFromFloat(f float64) *time.Time {
sec := int64(f)
nsec := int64((f - float64(sec)) * 1e9)
t := time.Unix(sec, nsec).UTC()
return &t
}
// itemToDocument mirrors _item_to_document.
func (c *RestAPIConnector) itemToDocument(item map[string]any) (SourceDocument, error) {
var rawID string
if c.cfg.IDField != "" {
if v := c.getTypedFieldValue(c.cfg.IDField, item); v != nil {
rawID = coerceRestAPIToText(v)
} else {
rawID = restAPIHash128("rest_api_item:" + restAPIStableText(item))
}
} else {
rawID = restAPIHash128("rest_api_item:" + restAPIStableText(item))
}
docID := restAPIHash128("rest_api:" + rawID)
var contentText string
if c.cfg.ContentTemplate != "" {
contentText = c.renderContentTemplate(item)
} else {
parts := []string{}
for _, field := range c.cfg.ContentFields {
if v := c.getTypedFieldValue(field, item); v != nil {
if text := restAPIStripHTML(coerceRestAPIToText(v)); text != "" {
parts = append(parts, text)
}
}
}
contentText = strings.Join(parts, "\n")
}
blob := []byte(contentText)
var metadata map[string]any
if len(c.cfg.MetadataFields) > 0 {
metadata = map[string]any{}
for _, field := range c.cfg.MetadataFields {
if v := c.getTypedFieldValue(field, item); v != nil {
metadata[field] = serializeRestAPIMetadataValue(v)
}
}
}
updatedAt := c.extractTimestamp(item)
if updatedAt == nil {
now := time.Now().UTC()
updatedAt = &now
}
sem := rawID
if len(c.cfg.ContentFields) > 0 {
sem = restAPIValueText(extractRestAPIField(item, c.cfg.ContentFields[0]))
}
sem = restAPIStripHTML(sem)
sem = strings.ReplaceAll(sem, "\n", " ")
sem = strings.ReplaceAll(sem, "\r", " ")
sem = strings.TrimSpace(sem)
if len(sem) > 100 {
sem = sem[:100]
}
if sem == "" {
sem = docID
}
return SourceDocument{
SourceID: docID,
SemanticIdentifier: sem,
Extension: ".txt",
Blob: blob,
UpdatedAt: *updatedAt,
SizeBytes: int64(len(blob)),
Metadata: metadata,
Fingerprint: contentFingerprint(blob),
}, nil
}
// renderContentTemplate mirrors _render_content_template.
func (c *RestAPIConnector) renderContentTemplate(item map[string]any) string {
values := map[string]string{}
seen := map[string]bool{}
fields := append(append([]string{}, c.cfg.ContentFields...), c.cfg.MetadataFields...)
for _, field := range fields {
if seen[field] {
continue
}
seen[field] = true
if v := c.getTypedFieldValue(field, item); v != nil {
values[restAPIFieldToTemplateName(field)] = coerceRestAPIToText(v)
}
}
rendered, ok := renderRestAPITemplateSafeMap(c.cfg.ContentTemplate, values)
if !ok {
parts := []string{}
for _, field := range c.cfg.ContentFields {
if p := coerceRestAPIToText(c.getTypedFieldValue(field, item)); p != "" {
parts = append(parts, p)
}
}
rendered = strings.Join(parts, "\n")
}
return restAPIStripHTML(rendered)
}
var restAPITemplateIndexRE = regexp.MustCompile(`\[\d+\]|\[\*\]`)
func restAPIFieldToTemplateName(field string) string {
name := restAPITemplateIndexRE.ReplaceAllString(field, "")
return strings.ReplaceAll(name, ".", "_")
}
// renderRestAPITemplateSafeMap renders {name} placeholders from values with a
// safe lookup: unknown names become the empty string; malformed placeholders
// make the whole render fail so the caller falls back to joined content.
func renderRestAPITemplateSafeMap(template string, values map[string]string) (string, bool) {
var b strings.Builder
for i := 0; i < len(template); {
switch template[i] {
case '{':
if i+1 < len(template) && template[i+1] == '{' {
b.WriteByte('{')
i += 2
continue
}
end := strings.IndexByte(template[i:], '}')
if end < 0 {
return "", false
}
token := template[i+1 : i+end]
if !validRestAPITemplateKey(token) {
return "", false
}
b.WriteString(values[token])
i += end + 1
case '}':
if i+1 < len(template) && template[i+1] == '}' {
b.WriteByte('}')
i += 2
continue
}
return "", false
default:
b.WriteByte(template[i])
i++
}
}
return b.String(), true
}
func validRestAPITemplateKey(token string) bool {
if token == "" {
return false
}
for _, r := range token {
if r != '_' && !unicode.IsLetter(r) && !unicode.IsDigit(r) {
return false
}
}
return true
}
// restAPIHash128 mirrors api.utils.common.hash128 (xxhash128 hex digest).
func restAPIHash128(data string) string {
return contentFingerprint([]byte(data))
}
// restAPIStableText renders a JSON value as deterministic text: single quotes,
// True/False/None, and matching float formatting. Map keys are sorted to keep
// the fallback document ID stable across runs.
func restAPIStableText(value any) string {
switch typed := value.(type) {
case nil:
return "None"
case bool:
if typed {
return "True"
}
return "False"
case string:
return restAPIQuoteString(typed)
case json.Number:
if strings.ContainsAny(typed.String(), ".eE") {
if f, err := typed.Float64(); err == nil {
return restAPIFloatText(f)
}
}
return typed.String()
case int:
return strconv.Itoa(typed)
case int64:
return strconv.FormatInt(typed, 10)
case float64:
return restAPIFloatText(typed)
case []any:
parts := make([]string, 0, len(typed))
for _, item := range typed {
parts = append(parts, restAPIStableText(item))
}
return "[" + strings.Join(parts, ", ") + "]"
case map[string]any:
keys := make([]string, 0, len(typed))
for k := range typed {
keys = append(keys, k)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, k := range keys {
parts = append(parts, restAPIQuoteString(k)+": "+restAPIStableText(typed[k]))
}
return "{" + strings.Join(parts, ", ") + "}"
default:
return fmt.Sprint(value)
}
}
// restAPIQuoteString renders a string as a single-quoted literal with the
// standard backslash escapes; printable non-ASCII characters are kept as-is.
func restAPIQuoteString(s string) string {
var b strings.Builder
b.WriteByte('\'')
for _, r := range s {
switch r {
case '\\':
b.WriteString(`\\`)
case '\'':
b.WriteString(`\'`)
case '\n':
b.WriteString(`\n`)
case '\r':
b.WriteString(`\r`)
case '\t':
b.WriteString(`\t`)
case '\b':
b.WriteString(`\b`)
case '\f':
b.WriteString(`\f`)
default:
if r < 0x20 || r == 0x7f {
b.WriteString(fmt.Sprintf(`\x%02x`, r))
} else {
b.WriteRune(r)
}
}
}
b.WriteByte('\'')
return b.String()
}
// restAPIValueText renders the scalar and composite JSON values the connector deals with
// as plain text.
func restAPIValueText(value any) string {
switch typed := value.(type) {
case nil:
return "None"
case bool:
if typed {
return "True"
}
return "False"
case string:
return typed
case json.Number:
if strings.ContainsAny(typed.String(), ".eE") {
if f, err := typed.Float64(); err == nil {
return restAPIFloatText(f)
}
}
return typed.String()
case int:
return strconv.Itoa(typed)
case int64:
return strconv.FormatInt(typed, 10)
case float64:
return restAPIFloatText(typed)
default:
return restAPIStableText(value)
}
}
// restAPIFloatText renders floats in a stable textual form.
func restAPIFloatText(f float64) string {
switch {
case math.IsInf(f, 1):
return "inf"
case math.IsInf(f, -1):
return "-inf"
case math.IsNaN(f):
return "nan"
case f == math.Trunc(f) && math.Abs(f) < 1e16:
return strconv.FormatFloat(f, 'f', 1, 64)
default:
return strconv.FormatFloat(f, 'g', -1, 64)
}
}
// coerceRestAPIToText mirrors _coerce_to_text.
func coerceRestAPIToText(value any) string {
switch typed := value.(type) {
case nil:
return ""
case string:
return typed
case json.Number:
if strings.ContainsAny(typed.String(), ".eE") {
if f, err := typed.Float64(); err == nil {
return restAPIFloatText(f)
}
}
return typed.String()
case int:
return strconv.Itoa(typed)
case int64:
return strconv.FormatInt(typed, 10)
case float64:
return restAPIFloatText(typed)
case bool:
if typed {
return "True"
}
return "False"
default:
return pyJSONDumps(value)
}
}
// serializeRestAPIMetadataValue mirrors _serialize_metadata_value.
func serializeRestAPIMetadataValue(value any) any {
switch typed := value.(type) {
case time.Time:
return restAPITimeISO(typed)
case json.Number:
if f, err := typed.Float64(); err == nil {
if f == math.Trunc(f) && !math.IsInf(f, 0) && math.Abs(f) < 1e15 {
return int64(f)
}
return f
}
return typed.String()
case int, int64, float64, bool, string:
return value
default:
return pyJSONDumps(value)
}
}
// pyJSONDumps mirrors json.dumps(value, ensure_ascii=False) with the default
// separators (", " and ": ").
func pyJSONDumps(value any) string {
var b strings.Builder
pyJSONDumpsTo(&b, value)
return b.String()
}
func pyJSONDumpsTo(b *strings.Builder, value any) {
switch typed := value.(type) {
case nil:
b.WriteString("null")
case bool:
if typed {
b.WriteString("true")
} else {
b.WriteString("false")
}
case string:
raw, _ := json.Marshal(typed)
b.Write(raw)
case json.Number:
if strings.ContainsAny(typed.String(), ".eE") {
if f, err := typed.Float64(); err == nil {
b.WriteString(restAPIFloatText(f))
return
}
}
b.WriteString(typed.String())
case int:
b.WriteString(strconv.Itoa(typed))
case int64:
b.WriteString(strconv.FormatInt(typed, 10))
case float64:
b.WriteString(restAPIFloatText(typed))
case []any:
b.WriteByte('[')
for i, item := range typed {
if i > 0 {
b.WriteString(", ")
}
pyJSONDumpsTo(b, item)
}
b.WriteByte(']')
case map[string]any:
keys := make([]string, 0, len(typed))
for k := range typed {
keys = append(keys, k)
}
sort.Strings(keys)
b.WriteByte('{')
for i, k := range keys {
if i > 0 {
b.WriteString(", ")
}
raw, _ := json.Marshal(k)
b.Write(raw)
b.WriteString(": ")
pyJSONDumpsTo(b, typed[k])
}
b.WriteByte('}')
default:
b.WriteString(fmt.Sprint(value))
}
}
var restAPITagRE = regexp.MustCompile(`<[^>]+>`)
var restAPIWhitespaceRE = regexp.MustCompile(`\s+`)
// restAPIStripHTML mirrors _strip_html.
func restAPIStripHTML(text string) string {
if !strings.Contains(text, "<") || !strings.Contains(text, ">") {
return text
}
cleaned := restAPITagRE.ReplaceAllString(text, " ")
return strings.TrimSpace(restAPIWhitespaceRE.ReplaceAllString(cleaned, " "))
}
// restAPITimeISO renders a UTC time as an ISO 8601 string, keeping fractional
// seconds when non-zero.
func restAPITimeISO(t time.Time) string {
t = t.UTC()
base := t.Format("2006-01-02T15:04:05")
if t.Nanosecond() != 0 {
base += fmt.Sprintf(".%06d", t.Nanosecond()/1000)
}
return base + "+00:00"
}
// ---------------------------------------------------------------------------
// Sync session
// ---------------------------------------------------------------------------
// restAPISyncCursor is the resume position serialized into SyncCheckpoint.Cursor.
// It stores the current page to fetch and the source document that was last
// committed from that page; pagination_type=none never produces a checkpoint.
type restAPISyncCursor struct {
Page int `json:"page,omitempty"`
Offset int `json:"offset,omitempty"`
Cursor string `json:"cursor,omitempty"`
SourceID string `json:"source_id,omitempty"`
}
// restAPIBufferedDocument carries the checkpoint position for an individual
// source document so a retried task can resume at the last emitted document.
type restAPIBufferedDocument struct {
document SourceDocument
checkpoint *SyncCheckpoint
}
// restAPISyncSession streams items lazily so large APIs are not fully
// materialized in memory.
type restAPISyncSession struct {
connector *RestAPIConnector
iterator *restAPIItemIterator
batchSize int
fromBeginning bool
windowStart *time.Time
windowEnd time.Time
pending []restAPIBufferedDocument
resumePosition *restAPISyncCursor
resumeSourceID string
}
// applyResume restores the checkpoint page and source anchor. Cursors that are
// malformed, mismatch the configured pagination type, or carry no source anchor
// are invalid because resuming would silently skip changed data.
func (s *restAPISyncSession) applyResume(checkpoint *SyncCheckpoint) error {
if checkpoint == nil || checkpoint.Cursor == "" {
if checkpoint == nil {
return nil
}
return fmt.Errorf("rest api sync cursor is missing: %w", ErrSyncResumeInvalid)
}
var cursor restAPISyncCursor
if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
return fmt.Errorf("rest api sync cursor is invalid: %w", ErrSyncResumeInvalid)
}
sourceID := firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
if sourceID == "" {
return fmt.Errorf("rest api sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
switch s.connector.cfg.PaginationType {
case restAPIPaginationPage:
if cursor.Page <= 0 || cursor.Offset != 0 || cursor.Cursor != "" {
return fmt.Errorf("rest api sync cursor does not match page pagination: %w", ErrSyncResumeInvalid)
}
s.iterator.page = cursor.Page
case restAPIPaginationOffset:
if cursor.Page != 0 || cursor.Cursor != "" {
return fmt.Errorf("rest api sync cursor does not match offset pagination: %w", ErrSyncResumeInvalid)
}
s.iterator.offset = cursor.Offset
case restAPIPaginationCursor:
if cursor.Page != 0 || cursor.Offset != 0 {
return fmt.Errorf("rest api sync cursor does not match cursor pagination: %w", ErrSyncResumeInvalid)
}
s.iterator.cursor = cursor.Cursor
default:
return fmt.Errorf("rest api sync checkpoint cannot resume pagination_type=none: %w", ErrSyncResumeInvalid)
}
position := cursor
position.SourceID = ""
s.resumePosition = &position
s.resumeSourceID = sourceID
return nil
}
// filterResumedDocuments drops every document at or before the checkpoint anchor.
// The anchor must still exist on the checkpoint page, otherwise the listing has
// changed and the runner should restart the whole fixed window.
func (s *restAPISyncSession) filterResumedDocuments(candidates []SourceDocument) ([]SourceDocument, error) {
if s.resumeSourceID == "" {
return candidates, nil
}
if s.resumePosition == nil || !restAPISyncCursorPositionEqual(s.iterator.currentPagePosition(), s.resumePosition) {
return nil, fmt.Errorf("rest api sync resume page no longer matches checkpoint: %w", ErrSyncResumeInvalid)
}
for index, doc := range candidates {
if doc.SourceID == s.resumeSourceID {
s.resumePosition = nil
s.resumeSourceID = ""
return candidates[index+1:], nil
}
}
return nil, fmt.Errorf("rest api resume anchor %q was not found on the checkpoint page: %w", s.resumeSourceID, ErrSyncResumeInvalid)
}
func restAPISyncCursorPositionEqual(left, right *restAPISyncCursor) bool {
if left == nil || right == nil {
return false
}
return left.Page == right.Page && left.Offset == right.Offset && left.Cursor == right.Cursor
}
// restAPISyncCheckpoint records the current page position plus the document
// that was last emitted from it.
func restAPISyncCheckpoint(position *restAPISyncCursor, doc SourceDocument) *SyncCheckpoint {
if position == nil {
return nil
}
cursor := *position
cursor.SourceID = doc.SourceID
raw, err := json.Marshal(cursor)
if err != nil {
return nil
}
updatedAt := doc.UpdatedAt.UTC()
return &SyncCheckpoint{
Cursor: string(raw),
SourceID: doc.SourceID,
UpdatedAt: &updatedAt,
}
}
// NextBatch returns the next batch of source documents. The batch checkpoint
// points back to the page of its last document, so a retried task re-fetches
// that page and skips through the saved source anchor.
func (s *restAPISyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
for len(s.pending) < s.batchSize {
items, err := s.iterator.nextPage(ctx)
if err != nil {
return SyncBatch{}, err
}
if items == nil {
break
}
candidates := make([]SourceDocument, 0, len(items))
for _, item := range items {
doc, err := s.connector.itemToDocument(item)
if err != nil {
// Items that fail conversion are skipped with a warning.
continue
}
if !s.fromBeginning && s.windowStart != nil {
if !restAPIDocInTimeWindow(doc.UpdatedAt, *s.windowStart, s.windowEnd) {
continue
}
}
candidates = append(candidates, doc)
}
candidates, err = s.filterResumedDocuments(candidates)
if err != nil {
return SyncBatch{}, err
}
position := s.iterator.currentPagePosition()
for _, doc := range candidates {
s.pending = append(s.pending, restAPIBufferedDocument{
document: doc,
checkpoint: restAPISyncCheckpoint(position, doc),
})
}
}
if len(s.pending) == 0 {
return SyncBatch{}, io.EOF
}
end := s.batchSize
if end > len(s.pending) {
end = len(s.pending)
}
documents := make([]SourceDocument, 0, end)
var checkpoint *SyncCheckpoint
for _, buffered := range s.pending[:end] {
documents = append(documents, buffered.document)
if buffered.checkpoint != nil {
checkpoint = buffered.checkpoint
}
}
s.pending = s.pending[end:]
return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil
}
// Close releases the session.
func (s *restAPISyncSession) Close() error {
return nil
}
// restAPIDocInTimeWindow mirrors _doc_in_time_window: start <= dt < end.
func restAPIDocInTimeWindow(docUpdatedAt, start, end time.Time) bool {
dt := docUpdatedAt.UTC()
return !dt.Before(start) && dt.Before(end)
}