RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix[syncer]: github data source don't validate repo's accessible (#18477)
Haruko38649d0d95f713c
diff --git a/internal/syncer/connector/github.go b/internal/syncer/connector/github.goindex 80d8f45fd1..2185fd60fb 100644--- a/internal/syncer/connector/github.go+++ b/internal/syncer/connector/github.go@@ -19,6 +19,7 @@ package connectorimport ("context""encoding/json"+ "errors""fmt""io""net/http"@@ -98,24 +99,116 @@ func (c *GitHubConnector) ValidateConnectorSetting(ctx context.Context, requestctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)defer cancel()if c == nil {- return fmt.Errorf("github connector is nil")- }- if c.owner == "" {- return fmt.Errorf("Invalid connector settings: 'repo_owner' must be provided")+ return &ConnectorValidationError{Message: "github connector is nil"}}if c.token == "" {- return fmt.Errorf("Missing github_access_token in credentials")+ return &ConnectorMissingCredentialError{Message: "Missing github_access_token in credentials"}+ }+ if c.owner == "" {+ return &ConnectorValidationError{Message: "Invalid connector settings: 'repo_owner' must be provided."}}if c.batchSize <= 0 {- return fmt.Errorf("batch_size must be a positive integer")+ return &ConnectorValidationError{Message: "batch_size must be a positive integer"}+ }++ return c.validateOwner(ctx)+}++// validateOwner validates the owner as an organization or a user with repositories.+func (c *GitHubConnector) validateOwner(ctx context.Context) error {+ err := c.validateOrgRepos(ctx)+ if err == nil {+ return nil+ }+ var apiErr *githubAPIError+ if errors.As(err, &apiErr) {+ if isGitHubMissingSSO(apiErr.Message) {+ const ssoGuideLink = "https://docs.github.com/en/enterprise-cloud@latest/authentication/authenticating-with-saml-single-sign-on/authorizing-a-personal-access-token-for-use-with-saml-single-sign-on"+ return &ConnectorValidationError{Message: fmt.Sprintf("Your GitHub token is missing authorization to access the `%s` organization. Please follow the guide to authorize your token: %s", c.owner, ssoGuideLink)}+ }+ if apiErr.Status != http.StatusNotFound {+ return classifyGitHubError(apiErr)+ }+ // The owner is not accessible as an organization; try as a user.+ userErr := c.validateUserRepos(ctx)+ if userErr == nil {+ return nil+ }+ var userAPIErr *githubAPIError+ if errors.As(userErr, &userAPIErr) && userAPIErr.Status == http.StatusNotFound {+ return &ConnectorValidationError{Message: fmt.Sprintf("GitHub user or organization not found: %s", c.owner)}+ }+ return classifyGitHubError(userErr)+ }+ return classifyGitHubError(err)+}++// validateOrgRepos validates the owner as a GitHub organization with repositories.+func (c *GitHubConnector) validateOrgRepos(ctx context.Context) error {+ var org map[string]any+ if _, err := c.getJSON(ctx, c.apiURL("/orgs/"+url.PathEscape(c.owner), nil), &org); err != nil {+ return err+ }+ hasRepos, err := c.ownerHasRepos(ctx, "/orgs/"+url.PathEscape(c.owner)+"/repos")+ if err != nil {+ return err+ }+ if !hasRepos {+ return &ConnectorValidationError{Message: fmt.Sprintf("Found no repos for organization: %s. Does the credential have the right scopes?", c.owner)}}+ return nil+}++// validateUserRepos validates the owner as a GitHub user with repositories.+func (c *GitHubConnector) validateUserRepos(ctx context.Context) error {var user map[string]any- if _, err := c.getJSON(ctx, c.apiURL("/user", nil), &user); err != nil {+ if _, err := c.getJSON(ctx, c.apiURL("/users/"+url.PathEscape(c.owner), nil), &user); err != nil {return err}+ hasRepos, err := c.ownerHasRepos(ctx, "/users/"+url.PathEscape(c.owner)+"/repos")+ if err != nil {+ return err+ }+ if !hasRepos {+ return &ConnectorValidationError{Message: fmt.Sprintf("Found no repos for user: %s. Does the credential have the right scopes?", c.owner)}+ }return nil}+// ownerHasRepos reports whether an org or user owner has at least one visible repository.+func (c *GitHubConnector) ownerHasRepos(ctx context.Context, path string) (bool, error) {+ query := url.Values{"per_page": {"1"}}+ var batch []githubRepo+ if _, err := c.getJSON(ctx, c.apiURL(path, query), &batch); err != nil {+ return false, err+ }+ return len(batch) > 0, nil+}++// classifyGitHubError maps a GitHub API error to a typed connector error.+func classifyGitHubError(err error) error {+ if err == nil {+ return nil+ }+ var apiErr *githubAPIError+ if !errors.As(err, &apiErr) {+ return err+ }+ switch apiErr.Status {+ case http.StatusUnauthorized:+ return &ConnectorMissingCredentialError{Message: "GitHub credential appears to be invalid or expired (HTTP 401)."}+ case http.StatusForbidden:+ return &ConnectorValidationError{Message: "Your GitHub token does not have sufficient permissions for this repository (HTTP 403)."}+ default:+ return &ConnectorValidationError{Message: fmt.Sprintf("Unexpected GitHub error (status=%d): %s", apiErr.Status, apiErr.Message)}+ }+}++// isGitHubMissingSSO reports whether a GitHub API error body requires organization SSO authorization.+func isGitHubMissingSSO(body string) bool {+ return strings.Contains(strings.ToLower(body), "you must grant your personal access token access to this organization")+}+// OpenSync opens one GitHub sync session.func (c *GitHubConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {repos, err := c.listRepos(ctx)@@ -309,7 +402,7 @@ func (c *GitHubConnector) getJSON(ctx context.Context, apiURL string, out any) (defer resp.Body.Close()if resp.StatusCode >= 400 {body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))- return nil, fmt.Errorf("GitHub API returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))+ return nil, &githubAPIError{Status: resp.StatusCode, Message: strings.TrimSpace(string(body))}}if err = json.NewDecoder(resp.Body).Decode(out); err != nil {return nil, err@@ -624,6 +717,19 @@ type githubRepo struct {FullName string `json:"full_name"`}+// githubAPIError is a GitHub API error with its HTTP status.+type githubAPIError struct {+ Status int+ Message string+}++func (e *githubAPIError) Error() string {+ if e.Status == 0 {+ return e.Message+ }+ return fmt.Sprintf("GitHub API returned HTTP %d: %s", e.Status, e.Message)+}+type githubSyncCursor struct {Repo string `json:"repo"`Stage string `json:"stage"`diff --git a/internal/syncer/connector/github_test.go b/internal/syncer/connector/github_test.goindex afd40ede6d..ad386b13bf 100644--- a/internal/syncer/connector/github_test.go+++ b/internal/syncer/connector/github_test.go@@ -7,6 +7,7 @@ import ("io""net/http""net/url"+ "strings""testing""time")@@ -300,3 +301,232 @@ func githubFixtureDoJSON(t *testing.T) func(ctx context.Context, apiURL string,return http.Header{}, nil}}++// githubValidationDoJSON returns a GitHub fixture transport for connection testing.+func githubValidationDoJSON(t *testing.T, ok map[string]string, fail map[string]*githubAPIError) func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ t.Helper()+ return func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ parsed, err := url.Parse(apiURL)+ if err != nil {+ t.Fatalf("parse api url: %v", err)+ }+ if strings.HasPrefix(parsed.Path, "/repos/") || strings.Contains(parsed.Path, "/contents") {+ t.Fatalf("ValidateConnectorSetting should not call repository or contents endpoint: %s", parsed.Path)+ }+ if apiErr, exists := fail[parsed.Path]; exists {+ return nil, apiErr+ }+ body, exists := ok[parsed.Path]+ if !exists {+ t.Fatalf("unexpected api path %s", parsed.Path)+ }+ if err = json.Unmarshal([]byte(body), out); err != nil {+ t.Fatalf("decode fixture: %v", err)+ }+ return http.Header{}, nil+ }+}++// TestGitHubValidateConnectorSettingSingleRepoSkipsRepoAccess verifies test+// connection does not probe repository or contents endpoints.+func TestGitHubValidateConnectorSettingSingleRepoSkipsRepoAccess(t *testing.T) {+ connector, err := NewGitHubConnector(map[string]any{+ "repository_owner": "openai",+ "repository_name": "ragflow",+ "credentials": map[string]any{"github_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitHubConnector failed: %v", err)+ }+ connector.baseURL = "https://api.github.test"+ connector.doJSON = githubValidationDoJSON(t,+ map[string]string{+ "/orgs/openai": `{"login":"openai"}`,+ "/orgs/openai/repos": `[{"full_name":"openai/ragflow"}]`,+ },+ nil,+ )+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {+ t.Fatalf("ValidateConnectorSetting: %v", err)+ }+}++// TestGitHubValidateConnectorSettingMultipleReposSkipsRepoAccess verifies comma-separated repository names+// do not trigger repository-level access probes during test connection.+func TestGitHubValidateConnectorSettingMultipleReposSkipsRepoAccess(t *testing.T) {+ connector, err := NewGitHubConnector(map[string]any{+ "repository_owner": "openai",+ "repository_name": "missing, ragflow",+ "credentials": map[string]any{"github_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitHubConnector failed: %v", err)+ }+ connector.baseURL = "https://api.github.test"+ connector.doJSON = githubValidationDoJSON(t,+ map[string]string{+ "/orgs/openai": `{"login":"openai"}`,+ "/orgs/openai/repos": `[{"full_name":"openai/ragflow"}]`,+ },+ nil,+ )+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {+ t.Fatalf("ValidateConnectorSetting: %v", err)+ }+}++// TestGitHubValidateConnectorSettingUnauthorized verifies an invalid token is detected.+func TestGitHubValidateConnectorSettingUnauthorized(t *testing.T) {+ connector, err := NewGitHubConnector(map[string]any{+ "repository_owner": "openai",+ "repository_name": "ragflow",+ "credentials": map[string]any{"github_access_token": "bad-token"},+ })+ if err != nil {+ t.Fatalf("NewGitHubConnector failed: %v", err)+ }+ connector.baseURL = "https://api.github.test"+ connector.doJSON = githubValidationDoJSON(t,+ nil,+ map[string]*githubAPIError{+ "/orgs/openai": {Status: http.StatusUnauthorized, Message: `{"message":"Bad credentials"}`},+ },+ )+ err = connector.ValidateConnectorSetting(context.Background(), nil)+ var credErr *ConnectorMissingCredentialError+ if !errors.As(err, &credErr) {+ t.Fatalf("err = %v, want ConnectorMissingCredentialError", err)+ }+}++// TestGitHubValidateConnectorSettingOwnerOrg verifies an owner with repositories passes.+func TestGitHubValidateConnectorSettingOwnerOrg(t *testing.T) {+ connector, err := NewGitHubConnector(map[string]any{+ "repository_owner": "openai",+ "credentials": map[string]any{"github_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitHubConnector failed: %v", err)+ }+ connector.baseURL = "https://api.github.test"+ connector.doJSON = githubValidationDoJSON(t,+ map[string]string{+ "/orgs/openai": `{"login":"openai"}`,+ "/orgs/openai/repos": `[{"full_name":"openai/ragflow"}]`,+ },+ nil,+ )+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {+ t.Fatalf("ValidateConnectorSetting: %v", err)+ }+}++// TestGitHubValidateConnectorSettingOwnerUserFallback verifies a user owner falls back from org lookup.+func TestGitHubValidateConnectorSettingOwnerUserFallback(t *testing.T) {+ connector, err := NewGitHubConnector(map[string]any{+ "repository_owner": "openai",+ "credentials": map[string]any{"github_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitHubConnector failed: %v", err)+ }+ connector.baseURL = "https://api.github.test"+ connector.doJSON = githubValidationDoJSON(t,+ map[string]string{+ "/users/openai": `{"login":"openai"}`,+ "/users/openai/repos": `[{"full_name":"openai/ragflow"}]`,+ },+ map[string]*githubAPIError{+ "/orgs/openai": {Status: http.StatusNotFound, Message: `{"message":"Not Found"}`},+ },+ )+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {+ t.Fatalf("ValidateConnectorSetting: %v", err)+ }+}++// TestGitHubValidateConnectorSettingOwnerNotFound verifies an unknown owner fails.+func TestGitHubValidateConnectorSettingOwnerNotFound(t *testing.T) {+ connector, err := NewGitHubConnector(map[string]any{+ "repository_owner": "ghost",+ "credentials": map[string]any{"github_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitHubConnector failed: %v", err)+ }+ connector.baseURL = "https://api.github.test"+ connector.doJSON = githubValidationDoJSON(t,+ nil,+ map[string]*githubAPIError{+ "/orgs/ghost": {Status: http.StatusNotFound, Message: `{"message":"Not Found"}`},+ "/users/ghost": {Status: http.StatusNotFound, Message: `{"message":"Not Found"}`},+ },+ )+ err = connector.ValidateConnectorSetting(context.Background(), nil)+ var valErr *ConnectorValidationError+ if !errors.As(err, &valErr) {+ t.Fatalf("err = %v, want ConnectorValidationError", err)+ }+ if got, want := valErr.Message, "GitHub user or organization not found: ghost"; got != want {+ t.Fatalf("message = %q, want %q", got, want)+ }+}++// TestGitHubValidateConnectorSettingMissingSSO verifies the SSO guide error is reported.+func TestGitHubValidateConnectorSettingMissingSSO(t *testing.T) {+ connector, err := NewGitHubConnector(map[string]any{+ "repository_owner": "ssoorg",+ "credentials": map[string]any{"github_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitHubConnector failed: %v", err)+ }+ connector.baseURL = "https://api.github.test"+ connector.doJSON = githubValidationDoJSON(t,+ nil,+ map[string]*githubAPIError{+ "/orgs/ssoorg": {Status: http.StatusForbidden, Message: `{"message":"Resource protected by organization SAML enforcement. You must grant your Personal Access token access to this organization."}`},+ },+ )+ err = connector.ValidateConnectorSetting(context.Background(), nil)+ var valErr *ConnectorValidationError+ if !errors.As(err, &valErr) {+ t.Fatalf("err = %v, want ConnectorValidationError", err)+ }+ if !strings.Contains(valErr.Message, "missing authorization") {+ t.Fatalf("message = %q, want SSO authorization hint", valErr.Message)+ }+}++// TestGitHubValidateConnectorSettingMissingInputs verifies missing config errors.+func TestGitHubValidateConnectorSettingMissingInputs(t *testing.T) {+ t.Run("missing token", func(t *testing.T) {+ connector, err := NewGitHubConnector(map[string]any{+ "repository_owner": "openai",+ "repository_name": "ragflow",+ "credentials": map[string]any{},+ })+ if err != nil {+ t.Fatalf("NewGitHubConnector failed: %v", err)+ }+ err = connector.ValidateConnectorSetting(context.Background(), nil)+ var credErr *ConnectorMissingCredentialError+ if !errors.As(err, &credErr) {+ t.Fatalf("err = %v, want ConnectorMissingCredentialError", err)+ }+ })+ t.Run("missing owner", func(t *testing.T) {+ connector, err := NewGitHubConnector(map[string]any{+ "repository_name": "ragflow",+ "credentials": map[string]any{"github_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitHubConnector failed: %v", err)+ }+ err = connector.ValidateConnectorSetting(context.Background(), nil)+ var valErr *ConnectorValidationError+ if !errors.As(err, &valErr) {+ t.Fatalf("err = %v, want ConnectorValidationError", err)+ }+ })+}