RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(connector): support file_extension for MySQL/PostgreSQL sync (#19466)
mkaaad0ff7e79c9574
diff --git a/internal/syncer/connector/fingerprint.go b/internal/syncer/connector/fingerprint.goindex 6c86506703..dcbc8152ab 100644--- a/internal/syncer/connector/fingerprint.go+++ b/internal/syncer/connector/fingerprint.go@@ -72,3 +72,14 @@ func syncSourceDocumentFilenameFromDocument(doc SourceDocument) string {}return name[:baseLimit] + ext}++// fileExtensionFromConfig returns the configured document file extension,+// defaulting to ".txt" when unset or blank. A missing leading dot is+// normalized downstream by syncSourceDocumentFilenameFromDocument.+func fileExtensionFromConfig(value any) string {+ ext := strings.TrimSpace(stringConfig(value))+ if ext == "" {+ return ".txt"+ }+ return ext+}diff --git a/internal/syncer/connector/mysql.go b/internal/syncer/connector/mysql.goindex fb5eecaad3..6e621183cd 100644--- a/internal/syncer/connector/mysql.go+++ b/internal/syncer/connector/mysql.go@@ -49,6 +49,7 @@ type MySQLConnector struct {metadataColumns []stringidColumn stringtimestampColumn string+ fileExtension stringbatchSize intusername stringpassword string@@ -65,6 +66,7 @@ func NewMySQLConnector(config map[string]any) (*MySQLConnector, error) {database: strings.TrimSpace(stringConfig(config["database"])),idColumn: strings.TrimSpace(stringConfig(config["id_column"])),timestampColumn: strings.TrimSpace(stringConfig(config["timestamp_column"])),+ fileExtension: fileExtensionFromConfig(config["file_extension"]),batchSize: configInt(config["batch_size"], defaultMySQLBatchSize),username: strings.TrimSpace(stringConfig(credentials["username"])),password: stringConfig(credentials["password"]),@@ -386,7 +388,7 @@ func (c *MySQLConnector) rowToSourceDocument(row map[string]any, orderedColumnsreturn SourceDocument{SourceID: sourceID,SemanticIdentifier: semanticID,- Extension: ".txt",+ Extension: c.fileExtension,Blob: blob,UpdatedAt: updatedAt,SizeBytes: int64(len(blob)),diff --git a/internal/syncer/connector/mysql_test.go b/internal/syncer/connector/mysql_test.goindex 42b1619b59..eaefb7c42d 100644--- a/internal/syncer/connector/mysql_test.go+++ b/internal/syncer/connector/mysql_test.go@@ -347,3 +347,71 @@ func TestMySQLConnectorStripOrderBy(t *testing.T) {}}}++// TestMySQLConnectorFileExtension verifies file_extension parsing and its+// ".txt" default.+func TestMySQLConnectorFileExtension(t *testing.T) {+ cases := []struct {+ name string+ config any+ want string+ }{+ {"default", nil, ".txt"},+ {"with dot", ".html", ".html"},+ {"no dot", "md", "md"},+ {"blank", " ", ".txt"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ connector, err := NewMySQLConnector(map[string]any{"file_extension": tc.config})+ if err != nil {+ t.Fatalf("NewMySQLConnector failed: %v", err)+ }+ if connector.fileExtension != tc.want {+ t.Fatalf("fileExtension = %q, want %q", connector.fileExtension, tc.want)+ }+ })+ }+}++// TestMySQLConnectorOpenSyncCustomFileExtension verifies a configured file+// extension is applied to produced documents.+func TestMySQLConnectorOpenSyncCustomFileExtension(t *testing.T) {+ query := "SELECT * FROM products WHERE status = 'active'"+ connector := newFixtureMySQLConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": "3306",+ "database": "mydb",+ "query": query,+ "content_columns": "title,description",+ "id_column": "id",+ "file_extension": ".html",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ mock.ExpectQuery(regexp.QuoteMeta(query)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "title", "description"}).+ AddRow(7, "Hello", "World"),+ )+ })++ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ if doc := batch.Documents[0]; doc.Extension != ".html" {+ t.Fatalf("extension = %q, want %q", doc.Extension, ".html")+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}diff --git a/internal/syncer/connector/postgresql.go b/internal/syncer/connector/postgresql.goindex c4bb70e70a..70d6dcdf29 100644--- a/internal/syncer/connector/postgresql.go+++ b/internal/syncer/connector/postgresql.go@@ -55,6 +55,7 @@ type PostgreSQLConnector struct {metadataColumns []stringidColumn stringtimestampColumn string+ fileExtension stringbatchSize intusername stringpassword string@@ -73,6 +74,7 @@ func NewPostgreSQLConnector(config map[string]any) (*PostgreSQLConnector, error)database: strings.TrimSpace(stringConfig(config["database"])),idColumn: strings.TrimSpace(stringConfig(config["id_column"])),timestampColumn: strings.TrimSpace(stringConfig(config["timestamp_column"])),+ fileExtension: fileExtensionFromConfig(config["file_extension"]),batchSize: configInt(config["batch_size"], defaultPostgresBatchSize),username: strings.TrimSpace(stringConfig(credentials["username"])),password: stringConfig(credentials["password"]),@@ -413,7 +415,7 @@ func (c *PostgreSQLConnector) rowToSourceDocument(row map[string]any, orderedColreturn SourceDocument{SourceID: sourceID,SemanticIdentifier: semanticID,- Extension: ".txt",+ Extension: c.fileExtension,Blob: blob,UpdatedAt: updatedAt,SizeBytes: int64(len(blob)),diff --git a/internal/syncer/connector/postgresql_test.go b/internal/syncer/connector/postgresql_test.goindex 4bb3be5dc9..253f2fc0d6 100644--- a/internal/syncer/connector/postgresql_test.go+++ b/internal/syncer/connector/postgresql_test.go@@ -338,3 +338,71 @@ func TestPostgreSQLConnectorValidate(t *testing.T) {t.Fatalf("Validate error = %v", err)}}++// TestPostgreSQLConnectorFileExtension verifies file_extension parsing and its+// ".txt" default.+func TestPostgreSQLConnectorFileExtension(t *testing.T) {+ cases := []struct {+ name string+ config any+ want string+ }{+ {"default", nil, ".txt"},+ {"with dot", ".html", ".html"},+ {"no dot", "md", "md"},+ {"blank", " ", ".txt"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ connector, err := NewPostgreSQLConnector(map[string]any{"file_extension": tc.config})+ if err != nil {+ t.Fatalf("NewPostgreSQLConnector failed: %v", err)+ }+ if connector.fileExtension != tc.want {+ t.Fatalf("fileExtension = %q, want %q", connector.fileExtension, tc.want)+ }+ })+ }+}++// TestPostgreSQLConnectorOpenSyncCustomFileExtension verifies a configured file+// extension is applied to produced documents.+func TestPostgreSQLConnectorOpenSyncCustomFileExtension(t *testing.T) {+ query := "SELECT * FROM products WHERE status = 'active'"+ connector := newFixturePostgresConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": "5432",+ "database": "mydb",+ "query": query,+ "content_columns": "title,description",+ "id_column": "id",+ "file_extension": ".md",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ mock.ExpectQuery(regexp.QuoteMeta(query)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "title", "description"}).+ AddRow(7, "Hello", "World"),+ )+ })++ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ if doc := batch.Documents[0]; doc.Extension != ".md" {+ t.Fatalf("extension = %q, want %q", doc.Extension, ".md")+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}