RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(syncer): accept rest api response without size limit (#21702)
cielc49a828e4e99
diff --git a/internal/syncer/connector/rest_api.go b/internal/syncer/connector/rest_api.goindex c9c5c84c5e..b25d1bb0ee 100644--- a/internal/syncer/connector/rest_api.go+++ b/internal/syncer/connector/rest_api.go@@ -17,6 +17,7 @@package connectorimport (+ "bytes""context""encoding/json""errors"@@ -87,6 +88,7 @@ const (restAPIDefaultRequestDelay = 0.5restAPIRequestTimeout = 60 * time.SecondrestAPIMaxRedirects = 5+ restAPIMaxResponseBytes = int64(32 * 1024 * 1024))const (@@ -727,11 +729,14 @@ func (c *RestAPIConnector) handleResponse(resp *http.Response) (any, error) {return nil, fmt.Errorf("REST API request failed with http %d", status)}- raw, err := io.ReadAll(resp.Body)+ raw, err := io.ReadAll(io.LimitReader(resp.Body, restAPIMaxResponseBytes+1))if err != nil {return nil, err}- decoder := json.NewDecoder(strings.NewReader(string(raw)))+ if int64(len(raw)) > restAPIMaxResponseBytes {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("REST API response exceeds the %d byte limit", restAPIMaxResponseBytes)}+ }+ decoder := json.NewDecoder(bytes.NewReader(raw))decoder.UseNumber()var value anyif err := decoder.Decode(&value); err != nil {