Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: secure Azure DevOps connector requests (#21509)

buua43620273da939db
diff --git a/internal/syncer/connector/azure_devops.go b/internal/syncer/connector/azure_devops.go
index 7bbcab9760..883f5ded62 100644
--- a/internal/syncer/connector/azure_devops.go
+++ b/internal/syncer/connector/azure_devops.go
@@ -20,6 +20,7 @@ import (
"context"
"encoding/base64"
"encoding/json"
+ "errors"
"fmt"
"io"
"net/http"
@@ -304,6 +305,9 @@ func (c *AzureDevOpsConnector) checkSettings() error {
if c.batchSize <= 0 {
return fmt.Errorf("batch_size must be a positive integer")
}
+ if _, _, err := assertConnectorURLSafeHTTPS(c.baseURL); err != nil {
+ return fmt.Errorf("Invalid connector settings: unsafe Azure DevOps URL: %w", err)
+ }
return nil
}
@@ -419,15 +423,26 @@ func (c *AzureDevOpsConnector) get(ctx context.Context, apiURL string, expectJSO
var lastErr error
for attempt := 0; attempt < azureDevOpsRetryTries; attempt++ {
- request, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
- if err != nil {
- return nil, err
+ timeout := 60 * time.Second
+ if c.httpClient != nil && c.httpClient.Timeout > 0 {
+ timeout = c.httpClient.Timeout
}
- request.Header.Set("Accept", "application/json")
- request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(":"+c.pat)))
-
- response, err := c.httpClient.Do(request)
+ response, err := connectorRequest(ctx, connectorRequestOptions{
+ Method: http.MethodGet,
+ RawURL: apiURL,
+ Headers: map[string]string{
+ "Accept": "application/json",
+ "Authorization": "Basic " + base64.StdEncoding.EncodeToString([]byte(":"+c.pat)),
+ },
+ Timeout: timeout,
+ Base: c.httpClient,
+ Validate: assertConnectorURLSafeHTTPS,
+ })
if err != nil {
+ var unsafeErr *connectorUnsafeURLError
+ if errors.As(err, &unsafeErr) {
+ return nil, unsafeErr.Err
+ }
lastErr = err
if sleepErr := sleepForAzureDevOps(ctx, delay); sleepErr != nil {
return nil, sleepErr
diff --git a/internal/syncer/connector/azure_devops_test.go b/internal/syncer/connector/azure_devops_test.go
index 56d865f9db..3311c97d97 100644
--- a/internal/syncer/connector/azure_devops_test.go
+++ b/internal/syncer/connector/azure_devops_test.go
@@ -26,6 +26,7 @@ import (
"net/http/httptest"
"strconv"
"strings"
+ "sync/atomic"
"testing"
"time"
)
@@ -33,6 +34,7 @@ import (
// newTestAzureDevOpsConnector points a connector at a stub server.
func newTestAzureDevOpsConnector(t *testing.T, serverURL string, overrides map[string]any) *AzureDevOpsConnector {
t.Helper()
+ withConnectorLoopbackTestHook(t)
config := map[string]any{
"organization": "contoso",
@@ -46,8 +48,8 @@ func newTestAzureDevOpsConnector(t *testing.T, serverURL string, overrides map[s
if err != nil {
t.Fatalf("NewAzureDevOpsConnector returned error: %v", err)
}
- // httptest serves over http, which the connector refuses for real
- // configuration, so the stub endpoint is injected directly.
+ // httptest serves over loopback; the test hook permits this local endpoint
+ // while keeping the production URL validation enabled.
connector.baseURL = serverURL
return connector
}
@@ -655,7 +657,7 @@ func TestAzureDevOpsLongPullRequestDescriptionIsRefetchedInFull(t *testing.T) {
}
}
-func TestAzureDevOpsAcceptsHTTPCollectionURLInClosedNetwork(t *testing.T) {
+func TestAzureDevOpsRejectsHTTPCollectionURL(t *testing.T) {
connector, err := NewAzureDevOpsConnector(map[string]any{
"organization": "http://tfs.contoso.com/DefaultCollection",
"credentials": map[string]any{"azure_devops_pat": "token"},
@@ -663,8 +665,70 @@ func TestAzureDevOpsAcceptsHTTPCollectionURLInClosedNetwork(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
- if connector.baseURL != "http://tfs.contoso.com/DefaultCollection" {
- t.Fatalf("expected HTTP collection URL, got %s", connector.baseURL)
+ if err := connector.checkSettings(); err == nil || !strings.Contains(err.Error(), "HTTPS") {
+ t.Fatalf("HTTP collection URL must be rejected, got %v", err)
+ }
+}
+
+func TestAzureDevOpsRejectsInternalURLs(t *testing.T) {
+ connectorAllowLoopbackForTest = false
+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })
+
+ testCases := []map[string]any{
+ {"base_url": "https://10.0.0.5/tfs", "organization": "DefaultCollection"},
+ {"base_url": "https://127.0.0.1/tfs", "organization": "DefaultCollection"},
+ {"organization": "https://169.254.169.254/DefaultCollection"},
+ {"organization": "https://[::1]/DefaultCollection"},
+ }
+ for _, config := range testCases {
+ config["credentials"] = map[string]any{"azure_devops_pat": "token"}
+ connector, err := NewAzureDevOpsConnector(config)
+ if err != nil {
+ t.Fatalf("NewAzureDevOpsConnector(%+v): %v", config, err)
+ }
+ if err := connector.checkSettings(); err == nil {
+ t.Errorf("internal Azure DevOps URL should be rejected: %+v", config)
+ }
+ }
+}
+
+func TestAzureDevOpsRejectsRedirectToPrivateURLWithoutRetry(t *testing.T) {
+ var attempts int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ atomic.AddInt32(&attempts, 1)
+ http.Redirect(w, r, "https://192.0.2.1/latest/meta-data/", http.StatusFound)
+ }))
+ defer server.Close()
+
+ connector := newTestAzureDevOpsConnector(t, server.URL, nil)
+ _, err := connector.get(context.Background(), server.URL+"/", true, 0)
+ if err == nil {
+ t.Fatal("redirect to a private address should be rejected")
+ }
+ if got := atomic.LoadInt32(&attempts); got != 1 {
+ t.Fatalf("unsafe redirects must not be retried, got %d source requests", got)
+ }
+}
+
+func TestAzureDevOpsRedirectDoesNotForwardPATAcrossOrigins(t *testing.T) {
+ receivedAuthorization := make(chan string, 1)
+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ receivedAuthorization <- r.Header.Get("Authorization")
+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{}})
+ }))
+ defer target.Close()
+
+ source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Redirect(w, r, target.URL+"/projects", http.StatusFound)
+ }))
+ defer source.Close()
+
+ connector := newTestAzureDevOpsConnector(t, source.URL, nil)
+ if _, err := connector.get(context.Background(), source.URL+"/", true, 0); err != nil {
+ t.Fatalf("request following a safe redirect failed: %v", err)
+ }
+ if authorization := <-receivedAuthorization; authorization != "" {
+ t.Fatalf("PAT must not be forwarded to another origin, got %q", authorization)
}
}
@@ -827,60 +891,25 @@ func TestAzureDevOpsActivePullRequestIsAlwaysReindexed(t *testing.T) {
}
}
-// countingBody reports how many bytes the connector actually pulled off the
-// wire, so a regression that reads everything and checks the size afterwards
-// cannot pass.
-type countingBody struct {
- remaining int64
- read *int64
-}
-
-func (b *countingBody) Read(p []byte) (int, error) {
- if b.remaining <= 0 {
- return 0, io.EOF
- }
- n := int64(len(p))
- if n > b.remaining {
- n = b.remaining
- }
- for i := int64(0); i < n; i++ {
- p[i] = 'x'
- }
- b.remaining -= n
- *b.read += n
- return int(n), nil
-}
-
-func (b *countingBody) Close() error { return nil }
-
-type stubTransport struct {
- respond func(*http.Request) *http.Response
-}
-
-func (t stubTransport) RoundTrip(request *http.Request) (*http.Response, error) {
- return t.respond(request), nil
-}
-
// A single oversized repository file must never be allocated in full.
func TestAzureDevOpsOversizedFileStopsAtTheLimit(t *testing.T) {
- var read int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/plain")
+ _, _ = io.WriteString(w, strings.Repeat("x", azureDevOpsMaxFileBytes*4))
+ }))
+ defer server.Close()
- connector, err := NewAzureDevOpsConnector(map[string]any{
- "organization": "contoso",
- "credentials": map[string]any{"azure_devops_pat": "token"},
- })
+ connector := newTestAzureDevOpsConnector(t, server.URL, nil)
+
+ repo := azureDevOpsRepository{Project: "iddaa", Name: "repo-a", Branch: "master"}
+ body, err := connector.get(context.Background(), server.URL+"/", false, azureDevOpsMaxFileBytes)
if err != nil {
- t.Fatalf("unexpected error: %v", err)
+ t.Fatalf("bounded response read failed: %v", err)
+ }
+ if len(body) != azureDevOpsMaxFileBytes+1 {
+ t.Fatalf("response read should stop one byte past the limit, got %d bytes", len(body))
}
- connector.httpClient = &http.Client{Transport: stubTransport{respond: func(*http.Request) *http.Response {
- return &http.Response{
- StatusCode: http.StatusOK,
- Header: http.Header{"Content-Type": []string{"text/plain"}},
- Body: &countingBody{remaining: azureDevOpsMaxFileBytes * 4, read: &read},
- }
- }}}
- repo := azureDevOpsRepository{Project: "iddaa", Name: "repo-a", Branch: "master"}
content, err := connector.fetchFile(context.Background(), repo, "/huge.bin")
if err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -888,9 +917,6 @@ func TestAzureDevOpsOversizedFileStopsAtTheLimit(t *testing.T) {
if content != nil {
t.Fatalf("an oversized file must be skipped, got %d bytes", len(content))
}
- if read != azureDevOpsMaxFileBytes+1 {
- t.Fatalf("the download must stop one byte past the limit, read %d bytes", read)
- }
}
func TestAzureDevOpsFileWithinLimitIsReturned(t *testing.T) {