Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(connector): support file_extension for MySQL/PostgreSQL sync (#19466)

mkaaad0ff7e79c9574
diff --git a/internal/syncer/connector/fingerprint.go b/internal/syncer/connector/fingerprint.go
index 6c86506703..dcbc8152ab 100644
--- a/internal/syncer/connector/fingerprint.go
+++ b/internal/syncer/connector/fingerprint.go
@@ -72,3 +72,14 @@ func syncSourceDocumentFilenameFromDocument(doc SourceDocument) string {
}
return name[:baseLimit] + ext
}
+
+// fileExtensionFromConfig returns the configured document file extension,
+// defaulting to ".txt" when unset or blank. A missing leading dot is
+// normalized downstream by syncSourceDocumentFilenameFromDocument.
+func fileExtensionFromConfig(value any) string {
+ ext := strings.TrimSpace(stringConfig(value))
+ if ext == "" {
+ return ".txt"
+ }
+ return ext
+}
diff --git a/internal/syncer/connector/mysql.go b/internal/syncer/connector/mysql.go
index fb5eecaad3..6e621183cd 100644
--- a/internal/syncer/connector/mysql.go
+++ b/internal/syncer/connector/mysql.go
@@ -49,6 +49,7 @@ type MySQLConnector struct {
metadataColumns []string
idColumn string
timestampColumn string
+ fileExtension string
batchSize int
username string
password string
@@ -65,6 +66,7 @@ func NewMySQLConnector(config map[string]any) (*MySQLConnector, error) {
database: strings.TrimSpace(stringConfig(config["database"])),
idColumn: strings.TrimSpace(stringConfig(config["id_column"])),
timestampColumn: strings.TrimSpace(stringConfig(config["timestamp_column"])),
+ fileExtension: fileExtensionFromConfig(config["file_extension"]),
batchSize: configInt(config["batch_size"], defaultMySQLBatchSize),
username: strings.TrimSpace(stringConfig(credentials["username"])),
password: stringConfig(credentials["password"]),
@@ -386,7 +388,7 @@ func (c *MySQLConnector) rowToSourceDocument(row map[string]any, orderedColumns
return SourceDocument{
SourceID: sourceID,
SemanticIdentifier: semanticID,
- Extension: ".txt",
+ Extension: c.fileExtension,
Blob: blob,
UpdatedAt: updatedAt,
SizeBytes: int64(len(blob)),
diff --git a/internal/syncer/connector/mysql_test.go b/internal/syncer/connector/mysql_test.go
index 42b1619b59..eaefb7c42d 100644
--- a/internal/syncer/connector/mysql_test.go
+++ b/internal/syncer/connector/mysql_test.go
@@ -347,3 +347,71 @@ func TestMySQLConnectorStripOrderBy(t *testing.T) {
}
}
}
+
+// TestMySQLConnectorFileExtension verifies file_extension parsing and its
+// ".txt" default.
+func TestMySQLConnectorFileExtension(t *testing.T) {
+ cases := []struct {
+ name string
+ config any
+ want string
+ }{
+ {"default", nil, ".txt"},
+ {"with dot", ".html", ".html"},
+ {"no dot", "md", "md"},
+ {"blank", " ", ".txt"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ connector, err := NewMySQLConnector(map[string]any{"file_extension": tc.config})
+ if err != nil {
+ t.Fatalf("NewMySQLConnector failed: %v", err)
+ }
+ if connector.fileExtension != tc.want {
+ t.Fatalf("fileExtension = %q, want %q", connector.fileExtension, tc.want)
+ }
+ })
+ }
+}
+
+// TestMySQLConnectorOpenSyncCustomFileExtension verifies a configured file
+// extension is applied to produced documents.
+func TestMySQLConnectorOpenSyncCustomFileExtension(t *testing.T) {
+ query := "SELECT * FROM products WHERE status = 'active'"
+ connector := newFixtureMySQLConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": "3306",
+ "database": "mydb",
+ "query": query,
+ "content_columns": "title,description",
+ "id_column": "id",
+ "file_extension": ".html",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ mock.ExpectQuery(regexp.QuoteMeta(query)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "title", "description"}).
+ AddRow(7, "Hello", "World"),
+ )
+ })
+
+ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+ if doc := batch.Documents[0]; doc.Extension != ".html" {
+ t.Fatalf("extension = %q, want %q", doc.Extension, ".html")
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
diff --git a/internal/syncer/connector/postgresql.go b/internal/syncer/connector/postgresql.go
index c4bb70e70a..70d6dcdf29 100644
--- a/internal/syncer/connector/postgresql.go
+++ b/internal/syncer/connector/postgresql.go
@@ -55,6 +55,7 @@ type PostgreSQLConnector struct {
metadataColumns []string
idColumn string
timestampColumn string
+ fileExtension string
batchSize int
username string
password string
@@ -73,6 +74,7 @@ func NewPostgreSQLConnector(config map[string]any) (*PostgreSQLConnector, error)
database: strings.TrimSpace(stringConfig(config["database"])),
idColumn: strings.TrimSpace(stringConfig(config["id_column"])),
timestampColumn: strings.TrimSpace(stringConfig(config["timestamp_column"])),
+ fileExtension: fileExtensionFromConfig(config["file_extension"]),
batchSize: configInt(config["batch_size"], defaultPostgresBatchSize),
username: strings.TrimSpace(stringConfig(credentials["username"])),
password: stringConfig(credentials["password"]),
@@ -413,7 +415,7 @@ func (c *PostgreSQLConnector) rowToSourceDocument(row map[string]any, orderedCol
return SourceDocument{
SourceID: sourceID,
SemanticIdentifier: semanticID,
- Extension: ".txt",
+ Extension: c.fileExtension,
Blob: blob,
UpdatedAt: updatedAt,
SizeBytes: int64(len(blob)),
diff --git a/internal/syncer/connector/postgresql_test.go b/internal/syncer/connector/postgresql_test.go
index 4bb3be5dc9..253f2fc0d6 100644
--- a/internal/syncer/connector/postgresql_test.go
+++ b/internal/syncer/connector/postgresql_test.go
@@ -338,3 +338,71 @@ func TestPostgreSQLConnectorValidate(t *testing.T) {
t.Fatalf("Validate error = %v", err)
}
}
+
+// TestPostgreSQLConnectorFileExtension verifies file_extension parsing and its
+// ".txt" default.
+func TestPostgreSQLConnectorFileExtension(t *testing.T) {
+ cases := []struct {
+ name string
+ config any
+ want string
+ }{
+ {"default", nil, ".txt"},
+ {"with dot", ".html", ".html"},
+ {"no dot", "md", "md"},
+ {"blank", " ", ".txt"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ connector, err := NewPostgreSQLConnector(map[string]any{"file_extension": tc.config})
+ if err != nil {
+ t.Fatalf("NewPostgreSQLConnector failed: %v", err)
+ }
+ if connector.fileExtension != tc.want {
+ t.Fatalf("fileExtension = %q, want %q", connector.fileExtension, tc.want)
+ }
+ })
+ }
+}
+
+// TestPostgreSQLConnectorOpenSyncCustomFileExtension verifies a configured file
+// extension is applied to produced documents.
+func TestPostgreSQLConnectorOpenSyncCustomFileExtension(t *testing.T) {
+ query := "SELECT * FROM products WHERE status = 'active'"
+ connector := newFixturePostgresConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": "5432",
+ "database": "mydb",
+ "query": query,
+ "content_columns": "title,description",
+ "id_column": "id",
+ "file_extension": ".md",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ mock.ExpectQuery(regexp.QuoteMeta(query)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "title", "description"}).
+ AddRow(7, "Hello", "World"),
+ )
+ })
+
+ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+ if doc := batch.Documents[0]; doc.Extension != ".md" {
+ t.Fatalf("extension = %q, want %q", doc.Extension, ".md")
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}