RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: secure Azure DevOps connector requests (#21509)
buua43620273da939db
diff --git a/internal/syncer/connector/azure_devops.go b/internal/syncer/connector/azure_devops.goindex 7bbcab9760..883f5ded62 100644--- a/internal/syncer/connector/azure_devops.go+++ b/internal/syncer/connector/azure_devops.go@@ -20,6 +20,7 @@ import ("context""encoding/base64""encoding/json"+ "errors""fmt""io""net/http"@@ -304,6 +305,9 @@ func (c *AzureDevOpsConnector) checkSettings() error {if c.batchSize <= 0 {return fmt.Errorf("batch_size must be a positive integer")}+ if _, _, err := assertConnectorURLSafeHTTPS(c.baseURL); err != nil {+ return fmt.Errorf("Invalid connector settings: unsafe Azure DevOps URL: %w", err)+ }return nil}@@ -419,15 +423,26 @@ func (c *AzureDevOpsConnector) get(ctx context.Context, apiURL string, expectJSOvar lastErr errorfor attempt := 0; attempt < azureDevOpsRetryTries; attempt++ {- request, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)- if err != nil {- return nil, err+ timeout := 60 * time.Second+ if c.httpClient != nil && c.httpClient.Timeout > 0 {+ timeout = c.httpClient.Timeout}- request.Header.Set("Accept", "application/json")- request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(":"+c.pat)))-- response, err := c.httpClient.Do(request)+ response, err := connectorRequest(ctx, connectorRequestOptions{+ Method: http.MethodGet,+ RawURL: apiURL,+ Headers: map[string]string{+ "Accept": "application/json",+ "Authorization": "Basic " + base64.StdEncoding.EncodeToString([]byte(":"+c.pat)),+ },+ Timeout: timeout,+ Base: c.httpClient,+ Validate: assertConnectorURLSafeHTTPS,+ })if err != nil {+ var unsafeErr *connectorUnsafeURLError+ if errors.As(err, &unsafeErr) {+ return nil, unsafeErr.Err+ }lastErr = errif sleepErr := sleepForAzureDevOps(ctx, delay); sleepErr != nil {return nil, sleepErrdiff --git a/internal/syncer/connector/azure_devops_test.go b/internal/syncer/connector/azure_devops_test.goindex 56d865f9db..3311c97d97 100644--- a/internal/syncer/connector/azure_devops_test.go+++ b/internal/syncer/connector/azure_devops_test.go@@ -26,6 +26,7 @@ import ("net/http/httptest""strconv""strings"+ "sync/atomic""testing""time")@@ -33,6 +34,7 @@ import (// newTestAzureDevOpsConnector points a connector at a stub server.func newTestAzureDevOpsConnector(t *testing.T, serverURL string, overrides map[string]any) *AzureDevOpsConnector {t.Helper()+ withConnectorLoopbackTestHook(t)config := map[string]any{"organization": "contoso",@@ -46,8 +48,8 @@ func newTestAzureDevOpsConnector(t *testing.T, serverURL string, overrides map[sif err != nil {t.Fatalf("NewAzureDevOpsConnector returned error: %v", err)}- // httptest serves over http, which the connector refuses for real- // configuration, so the stub endpoint is injected directly.+ // httptest serves over loopback; the test hook permits this local endpoint+ // while keeping the production URL validation enabled.connector.baseURL = serverURLreturn connector}@@ -655,7 +657,7 @@ func TestAzureDevOpsLongPullRequestDescriptionIsRefetchedInFull(t *testing.T) {}}-func TestAzureDevOpsAcceptsHTTPCollectionURLInClosedNetwork(t *testing.T) {+func TestAzureDevOpsRejectsHTTPCollectionURL(t *testing.T) {connector, err := NewAzureDevOpsConnector(map[string]any{"organization": "http://tfs.contoso.com/DefaultCollection","credentials": map[string]any{"azure_devops_pat": "token"},@@ -663,8 +665,70 @@ func TestAzureDevOpsAcceptsHTTPCollectionURLInClosedNetwork(t *testing.T) {if err != nil {t.Fatalf("unexpected error: %v", err)}- if connector.baseURL != "http://tfs.contoso.com/DefaultCollection" {- t.Fatalf("expected HTTP collection URL, got %s", connector.baseURL)+ if err := connector.checkSettings(); err == nil || !strings.Contains(err.Error(), "HTTPS") {+ t.Fatalf("HTTP collection URL must be rejected, got %v", err)+ }+}++func TestAzureDevOpsRejectsInternalURLs(t *testing.T) {+ connectorAllowLoopbackForTest = false+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })++ testCases := []map[string]any{+ {"base_url": "https://10.0.0.5/tfs", "organization": "DefaultCollection"},+ {"base_url": "https://127.0.0.1/tfs", "organization": "DefaultCollection"},+ {"organization": "https://169.254.169.254/DefaultCollection"},+ {"organization": "https://[::1]/DefaultCollection"},+ }+ for _, config := range testCases {+ config["credentials"] = map[string]any{"azure_devops_pat": "token"}+ connector, err := NewAzureDevOpsConnector(config)+ if err != nil {+ t.Fatalf("NewAzureDevOpsConnector(%+v): %v", config, err)+ }+ if err := connector.checkSettings(); err == nil {+ t.Errorf("internal Azure DevOps URL should be rejected: %+v", config)+ }+ }+}++func TestAzureDevOpsRejectsRedirectToPrivateURLWithoutRetry(t *testing.T) {+ var attempts int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ atomic.AddInt32(&attempts, 1)+ http.Redirect(w, r, "https://192.0.2.1/latest/meta-data/", http.StatusFound)+ }))+ defer server.Close()++ connector := newTestAzureDevOpsConnector(t, server.URL, nil)+ _, err := connector.get(context.Background(), server.URL+"/", true, 0)+ if err == nil {+ t.Fatal("redirect to a private address should be rejected")+ }+ if got := atomic.LoadInt32(&attempts); got != 1 {+ t.Fatalf("unsafe redirects must not be retried, got %d source requests", got)+ }+}++func TestAzureDevOpsRedirectDoesNotForwardPATAcrossOrigins(t *testing.T) {+ receivedAuthorization := make(chan string, 1)+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ receivedAuthorization <- r.Header.Get("Authorization")+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{}})+ }))+ defer target.Close()++ source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ http.Redirect(w, r, target.URL+"/projects", http.StatusFound)+ }))+ defer source.Close()++ connector := newTestAzureDevOpsConnector(t, source.URL, nil)+ if _, err := connector.get(context.Background(), source.URL+"/", true, 0); err != nil {+ t.Fatalf("request following a safe redirect failed: %v", err)+ }+ if authorization := <-receivedAuthorization; authorization != "" {+ t.Fatalf("PAT must not be forwarded to another origin, got %q", authorization)}}@@ -827,60 +891,25 @@ func TestAzureDevOpsActivePullRequestIsAlwaysReindexed(t *testing.T) {}}-// countingBody reports how many bytes the connector actually pulled off the-// wire, so a regression that reads everything and checks the size afterwards-// cannot pass.-type countingBody struct {- remaining int64- read *int64-}--func (b *countingBody) Read(p []byte) (int, error) {- if b.remaining <= 0 {- return 0, io.EOF- }- n := int64(len(p))- if n > b.remaining {- n = b.remaining- }- for i := int64(0); i < n; i++ {- p[i] = 'x'- }- b.remaining -= n- *b.read += n- return int(n), nil-}--func (b *countingBody) Close() error { return nil }--type stubTransport struct {- respond func(*http.Request) *http.Response-}--func (t stubTransport) RoundTrip(request *http.Request) (*http.Response, error) {- return t.respond(request), nil-}-// A single oversized repository file must never be allocated in full.func TestAzureDevOpsOversizedFileStopsAtTheLimit(t *testing.T) {- var read int64+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "text/plain")+ _, _ = io.WriteString(w, strings.Repeat("x", azureDevOpsMaxFileBytes*4))+ }))+ defer server.Close()- connector, err := NewAzureDevOpsConnector(map[string]any{- "organization": "contoso",- "credentials": map[string]any{"azure_devops_pat": "token"},- })+ connector := newTestAzureDevOpsConnector(t, server.URL, nil)++ repo := azureDevOpsRepository{Project: "iddaa", Name: "repo-a", Branch: "master"}+ body, err := connector.get(context.Background(), server.URL+"/", false, azureDevOpsMaxFileBytes)if err != nil {- t.Fatalf("unexpected error: %v", err)+ t.Fatalf("bounded response read failed: %v", err)+ }+ if len(body) != azureDevOpsMaxFileBytes+1 {+ t.Fatalf("response read should stop one byte past the limit, got %d bytes", len(body))}- connector.httpClient = &http.Client{Transport: stubTransport{respond: func(*http.Request) *http.Response {- return &http.Response{- StatusCode: http.StatusOK,- Header: http.Header{"Content-Type": []string{"text/plain"}},- Body: &countingBody{remaining: azureDevOpsMaxFileBytes * 4, read: &read},- }- }}}- repo := azureDevOpsRepository{Project: "iddaa", Name: "repo-a", Branch: "master"}content, err := connector.fetchFile(context.Background(), repo, "/huge.bin")if err != nil {t.Fatalf("unexpected error: %v", err)@@ -888,9 +917,6 @@ func TestAzureDevOpsOversizedFileStopsAtTheLimit(t *testing.T) {if content != nil {t.Fatalf("an oversized file must be skipped, got %d bytes", len(content))}- if read != azureDevOpsMaxFileBytes+1 {- t.Fatalf("the download must stop one byte past the limit, read %d bytes", read)- }}func TestAzureDevOpsFileWithinLimitIsReturned(t *testing.T) {