Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix[syncer]: github data source don't validate repo's accessible (#18477)

Haruko38649d0d95f713c
diff --git a/internal/syncer/connector/github.go b/internal/syncer/connector/github.go
index 80d8f45fd1..2185fd60fb 100644
--- a/internal/syncer/connector/github.go
+++ b/internal/syncer/connector/github.go
@@ -19,6 +19,7 @@ package connector
import (
"context"
"encoding/json"
+ "errors"
"fmt"
"io"
"net/http"
@@ -98,24 +99,116 @@ func (c *GitHubConnector) ValidateConnectorSetting(ctx context.Context, request
ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
defer cancel()
if c == nil {
- return fmt.Errorf("github connector is nil")
- }
- if c.owner == "" {
- return fmt.Errorf("Invalid connector settings: 'repo_owner' must be provided")
+ return &ConnectorValidationError{Message: "github connector is nil"}
}
if c.token == "" {
- return fmt.Errorf("Missing github_access_token in credentials")
+ return &ConnectorMissingCredentialError{Message: "Missing github_access_token in credentials"}
+ }
+ if c.owner == "" {
+ return &ConnectorValidationError{Message: "Invalid connector settings: 'repo_owner' must be provided."}
}
if c.batchSize <= 0 {
- return fmt.Errorf("batch_size must be a positive integer")
+ return &ConnectorValidationError{Message: "batch_size must be a positive integer"}
+ }
+
+ return c.validateOwner(ctx)
+}
+
+// validateOwner validates the owner as an organization or a user with repositories.
+func (c *GitHubConnector) validateOwner(ctx context.Context) error {
+ err := c.validateOrgRepos(ctx)
+ if err == nil {
+ return nil
+ }
+ var apiErr *githubAPIError
+ if errors.As(err, &apiErr) {
+ if isGitHubMissingSSO(apiErr.Message) {
+ const ssoGuideLink = "https://docs.github.com/en/enterprise-cloud@latest/authentication/authenticating-with-saml-single-sign-on/authorizing-a-personal-access-token-for-use-with-saml-single-sign-on"
+ return &ConnectorValidationError{Message: fmt.Sprintf("Your GitHub token is missing authorization to access the `%s` organization. Please follow the guide to authorize your token: %s", c.owner, ssoGuideLink)}
+ }
+ if apiErr.Status != http.StatusNotFound {
+ return classifyGitHubError(apiErr)
+ }
+ // The owner is not accessible as an organization; try as a user.
+ userErr := c.validateUserRepos(ctx)
+ if userErr == nil {
+ return nil
+ }
+ var userAPIErr *githubAPIError
+ if errors.As(userErr, &userAPIErr) && userAPIErr.Status == http.StatusNotFound {
+ return &ConnectorValidationError{Message: fmt.Sprintf("GitHub user or organization not found: %s", c.owner)}
+ }
+ return classifyGitHubError(userErr)
+ }
+ return classifyGitHubError(err)
+}
+
+// validateOrgRepos validates the owner as a GitHub organization with repositories.
+func (c *GitHubConnector) validateOrgRepos(ctx context.Context) error {
+ var org map[string]any
+ if _, err := c.getJSON(ctx, c.apiURL("/orgs/"+url.PathEscape(c.owner), nil), &org); err != nil {
+ return err
+ }
+ hasRepos, err := c.ownerHasRepos(ctx, "/orgs/"+url.PathEscape(c.owner)+"/repos")
+ if err != nil {
+ return err
+ }
+ if !hasRepos {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Found no repos for organization: %s. Does the credential have the right scopes?", c.owner)}
}
+ return nil
+}
+
+// validateUserRepos validates the owner as a GitHub user with repositories.
+func (c *GitHubConnector) validateUserRepos(ctx context.Context) error {
var user map[string]any
- if _, err := c.getJSON(ctx, c.apiURL("/user", nil), &user); err != nil {
+ if _, err := c.getJSON(ctx, c.apiURL("/users/"+url.PathEscape(c.owner), nil), &user); err != nil {
return err
}
+ hasRepos, err := c.ownerHasRepos(ctx, "/users/"+url.PathEscape(c.owner)+"/repos")
+ if err != nil {
+ return err
+ }
+ if !hasRepos {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Found no repos for user: %s. Does the credential have the right scopes?", c.owner)}
+ }
return nil
}
+// ownerHasRepos reports whether an org or user owner has at least one visible repository.
+func (c *GitHubConnector) ownerHasRepos(ctx context.Context, path string) (bool, error) {
+ query := url.Values{"per_page": {"1"}}
+ var batch []githubRepo
+ if _, err := c.getJSON(ctx, c.apiURL(path, query), &batch); err != nil {
+ return false, err
+ }
+ return len(batch) > 0, nil
+}
+
+// classifyGitHubError maps a GitHub API error to a typed connector error.
+func classifyGitHubError(err error) error {
+ if err == nil {
+ return nil
+ }
+ var apiErr *githubAPIError
+ if !errors.As(err, &apiErr) {
+ return err
+ }
+ switch apiErr.Status {
+ case http.StatusUnauthorized:
+ return &ConnectorMissingCredentialError{Message: "GitHub credential appears to be invalid or expired (HTTP 401)."}
+ case http.StatusForbidden:
+ return &ConnectorValidationError{Message: "Your GitHub token does not have sufficient permissions for this repository (HTTP 403)."}
+ default:
+ return &ConnectorValidationError{Message: fmt.Sprintf("Unexpected GitHub error (status=%d): %s", apiErr.Status, apiErr.Message)}
+ }
+}
+
+// isGitHubMissingSSO reports whether a GitHub API error body requires organization SSO authorization.
+func isGitHubMissingSSO(body string) bool {
+ return strings.Contains(strings.ToLower(body), "you must grant your personal access token access to this organization")
+}
+
// OpenSync opens one GitHub sync session.
func (c *GitHubConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
repos, err := c.listRepos(ctx)
@@ -309,7 +402,7 @@ func (c *GitHubConnector) getJSON(ctx context.Context, apiURL string, out any) (
defer resp.Body.Close()
if resp.StatusCode >= 400 {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
- return nil, fmt.Errorf("GitHub API returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+ return nil, &githubAPIError{Status: resp.StatusCode, Message: strings.TrimSpace(string(body))}
}
if err = json.NewDecoder(resp.Body).Decode(out); err != nil {
return nil, err
@@ -624,6 +717,19 @@ type githubRepo struct {
FullName string `json:"full_name"`
}
+// githubAPIError is a GitHub API error with its HTTP status.
+type githubAPIError struct {
+ Status int
+ Message string
+}
+
+func (e *githubAPIError) Error() string {
+ if e.Status == 0 {
+ return e.Message
+ }
+ return fmt.Sprintf("GitHub API returned HTTP %d: %s", e.Status, e.Message)
+}
+
type githubSyncCursor struct {
Repo string `json:"repo"`
Stage string `json:"stage"`
diff --git a/internal/syncer/connector/github_test.go b/internal/syncer/connector/github_test.go
index afd40ede6d..ad386b13bf 100644
--- a/internal/syncer/connector/github_test.go
+++ b/internal/syncer/connector/github_test.go
@@ -7,6 +7,7 @@ import (
"io"
"net/http"
"net/url"
+ "strings"
"testing"
"time"
)
@@ -300,3 +301,232 @@ func githubFixtureDoJSON(t *testing.T) func(ctx context.Context, apiURL string,
return http.Header{}, nil
}
}
+
+// githubValidationDoJSON returns a GitHub fixture transport for connection testing.
+func githubValidationDoJSON(t *testing.T, ok map[string]string, fail map[string]*githubAPIError) func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ t.Helper()
+ return func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ parsed, err := url.Parse(apiURL)
+ if err != nil {
+ t.Fatalf("parse api url: %v", err)
+ }
+ if strings.HasPrefix(parsed.Path, "/repos/") || strings.Contains(parsed.Path, "/contents") {
+ t.Fatalf("ValidateConnectorSetting should not call repository or contents endpoint: %s", parsed.Path)
+ }
+ if apiErr, exists := fail[parsed.Path]; exists {
+ return nil, apiErr
+ }
+ body, exists := ok[parsed.Path]
+ if !exists {
+ t.Fatalf("unexpected api path %s", parsed.Path)
+ }
+ if err = json.Unmarshal([]byte(body), out); err != nil {
+ t.Fatalf("decode fixture: %v", err)
+ }
+ return http.Header{}, nil
+ }
+}
+
+// TestGitHubValidateConnectorSettingSingleRepoSkipsRepoAccess verifies test
+// connection does not probe repository or contents endpoints.
+func TestGitHubValidateConnectorSettingSingleRepoSkipsRepoAccess(t *testing.T) {
+ connector, err := NewGitHubConnector(map[string]any{
+ "repository_owner": "openai",
+ "repository_name": "ragflow",
+ "credentials": map[string]any{"github_access_token": "token"},
+ })
+ if err != nil {
+ t.Fatalf("NewGitHubConnector failed: %v", err)
+ }
+ connector.baseURL = "https://api.github.test"
+ connector.doJSON = githubValidationDoJSON(t,
+ map[string]string{
+ "/orgs/openai": `{"login":"openai"}`,
+ "/orgs/openai/repos": `[{"full_name":"openai/ragflow"}]`,
+ },
+ nil,
+ )
+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+}
+
+// TestGitHubValidateConnectorSettingMultipleReposSkipsRepoAccess verifies comma-separated repository names
+// do not trigger repository-level access probes during test connection.
+func TestGitHubValidateConnectorSettingMultipleReposSkipsRepoAccess(t *testing.T) {
+ connector, err := NewGitHubConnector(map[string]any{
+ "repository_owner": "openai",
+ "repository_name": "missing, ragflow",
+ "credentials": map[string]any{"github_access_token": "token"},
+ })
+ if err != nil {
+ t.Fatalf("NewGitHubConnector failed: %v", err)
+ }
+ connector.baseURL = "https://api.github.test"
+ connector.doJSON = githubValidationDoJSON(t,
+ map[string]string{
+ "/orgs/openai": `{"login":"openai"}`,
+ "/orgs/openai/repos": `[{"full_name":"openai/ragflow"}]`,
+ },
+ nil,
+ )
+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+}
+
+// TestGitHubValidateConnectorSettingUnauthorized verifies an invalid token is detected.
+func TestGitHubValidateConnectorSettingUnauthorized(t *testing.T) {
+ connector, err := NewGitHubConnector(map[string]any{
+ "repository_owner": "openai",
+ "repository_name": "ragflow",
+ "credentials": map[string]any{"github_access_token": "bad-token"},
+ })
+ if err != nil {
+ t.Fatalf("NewGitHubConnector failed: %v", err)
+ }
+ connector.baseURL = "https://api.github.test"
+ connector.doJSON = githubValidationDoJSON(t,
+ nil,
+ map[string]*githubAPIError{
+ "/orgs/openai": {Status: http.StatusUnauthorized, Message: `{"message":"Bad credentials"}`},
+ },
+ )
+ err = connector.ValidateConnectorSetting(context.Background(), nil)
+ var credErr *ConnectorMissingCredentialError
+ if !errors.As(err, &credErr) {
+ t.Fatalf("err = %v, want ConnectorMissingCredentialError", err)
+ }
+}
+
+// TestGitHubValidateConnectorSettingOwnerOrg verifies an owner with repositories passes.
+func TestGitHubValidateConnectorSettingOwnerOrg(t *testing.T) {
+ connector, err := NewGitHubConnector(map[string]any{
+ "repository_owner": "openai",
+ "credentials": map[string]any{"github_access_token": "token"},
+ })
+ if err != nil {
+ t.Fatalf("NewGitHubConnector failed: %v", err)
+ }
+ connector.baseURL = "https://api.github.test"
+ connector.doJSON = githubValidationDoJSON(t,
+ map[string]string{
+ "/orgs/openai": `{"login":"openai"}`,
+ "/orgs/openai/repos": `[{"full_name":"openai/ragflow"}]`,
+ },
+ nil,
+ )
+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+}
+
+// TestGitHubValidateConnectorSettingOwnerUserFallback verifies a user owner falls back from org lookup.
+func TestGitHubValidateConnectorSettingOwnerUserFallback(t *testing.T) {
+ connector, err := NewGitHubConnector(map[string]any{
+ "repository_owner": "openai",
+ "credentials": map[string]any{"github_access_token": "token"},
+ })
+ if err != nil {
+ t.Fatalf("NewGitHubConnector failed: %v", err)
+ }
+ connector.baseURL = "https://api.github.test"
+ connector.doJSON = githubValidationDoJSON(t,
+ map[string]string{
+ "/users/openai": `{"login":"openai"}`,
+ "/users/openai/repos": `[{"full_name":"openai/ragflow"}]`,
+ },
+ map[string]*githubAPIError{
+ "/orgs/openai": {Status: http.StatusNotFound, Message: `{"message":"Not Found"}`},
+ },
+ )
+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+}
+
+// TestGitHubValidateConnectorSettingOwnerNotFound verifies an unknown owner fails.
+func TestGitHubValidateConnectorSettingOwnerNotFound(t *testing.T) {
+ connector, err := NewGitHubConnector(map[string]any{
+ "repository_owner": "ghost",
+ "credentials": map[string]any{"github_access_token": "token"},
+ })
+ if err != nil {
+ t.Fatalf("NewGitHubConnector failed: %v", err)
+ }
+ connector.baseURL = "https://api.github.test"
+ connector.doJSON = githubValidationDoJSON(t,
+ nil,
+ map[string]*githubAPIError{
+ "/orgs/ghost": {Status: http.StatusNotFound, Message: `{"message":"Not Found"}`},
+ "/users/ghost": {Status: http.StatusNotFound, Message: `{"message":"Not Found"}`},
+ },
+ )
+ err = connector.ValidateConnectorSetting(context.Background(), nil)
+ var valErr *ConnectorValidationError
+ if !errors.As(err, &valErr) {
+ t.Fatalf("err = %v, want ConnectorValidationError", err)
+ }
+ if got, want := valErr.Message, "GitHub user or organization not found: ghost"; got != want {
+ t.Fatalf("message = %q, want %q", got, want)
+ }
+}
+
+// TestGitHubValidateConnectorSettingMissingSSO verifies the SSO guide error is reported.
+func TestGitHubValidateConnectorSettingMissingSSO(t *testing.T) {
+ connector, err := NewGitHubConnector(map[string]any{
+ "repository_owner": "ssoorg",
+ "credentials": map[string]any{"github_access_token": "token"},
+ })
+ if err != nil {
+ t.Fatalf("NewGitHubConnector failed: %v", err)
+ }
+ connector.baseURL = "https://api.github.test"
+ connector.doJSON = githubValidationDoJSON(t,
+ nil,
+ map[string]*githubAPIError{
+ "/orgs/ssoorg": {Status: http.StatusForbidden, Message: `{"message":"Resource protected by organization SAML enforcement. You must grant your Personal Access token access to this organization."}`},
+ },
+ )
+ err = connector.ValidateConnectorSetting(context.Background(), nil)
+ var valErr *ConnectorValidationError
+ if !errors.As(err, &valErr) {
+ t.Fatalf("err = %v, want ConnectorValidationError", err)
+ }
+ if !strings.Contains(valErr.Message, "missing authorization") {
+ t.Fatalf("message = %q, want SSO authorization hint", valErr.Message)
+ }
+}
+
+// TestGitHubValidateConnectorSettingMissingInputs verifies missing config errors.
+func TestGitHubValidateConnectorSettingMissingInputs(t *testing.T) {
+ t.Run("missing token", func(t *testing.T) {
+ connector, err := NewGitHubConnector(map[string]any{
+ "repository_owner": "openai",
+ "repository_name": "ragflow",
+ "credentials": map[string]any{},
+ })
+ if err != nil {
+ t.Fatalf("NewGitHubConnector failed: %v", err)
+ }
+ err = connector.ValidateConnectorSetting(context.Background(), nil)
+ var credErr *ConnectorMissingCredentialError
+ if !errors.As(err, &credErr) {
+ t.Fatalf("err = %v, want ConnectorMissingCredentialError", err)
+ }
+ })
+ t.Run("missing owner", func(t *testing.T) {
+ connector, err := NewGitHubConnector(map[string]any{
+ "repository_name": "ragflow",
+ "credentials": map[string]any{"github_access_token": "token"},
+ })
+ if err != nil {
+ t.Fatalf("NewGitHubConnector failed: %v", err)
+ }
+ err = connector.ValidateConnectorSetting(context.Background(), nil)
+ var valErr *ConnectorValidationError
+ if !errors.As(err, &valErr) {
+ t.Fatalf("err = %v, want ConnectorValidationError", err)
+ }
+ })
+}