Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Custom the file extension when saving DB fields, and Parse HTML fragments not just full HTML doc (#15998)

Ren Wei (任卫)c9a6f92738bd
diff --git a/common/data_source/rdbms_connector.py b/common/data_source/rdbms_connector.py
index 4614d95875..c01bc1a11e 100644
--- a/common/data_source/rdbms_connector.py
+++ b/common/data_source/rdbms_connector.py
@@ -57,6 +57,7 @@ class RDBMSConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):
id_column: Optional[str] = None,
timestamp_column: Optional[str] = None,
batch_size: int = INDEX_BATCH_SIZE,
+ file_extension=None,
) -> None:
"""
Initialize the RDBMS connector.
@@ -72,6 +73,7 @@ class RDBMSConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):
id_column: Column to use as unique document ID (optional, will generate hash if not provided)
timestamp_column: Column to use for incremental sync (optional, must be datetime/timestamp type)
batch_size: Number of documents per batch
+ file_extension: File Extension to save content column into
"""
self.db_type = DatabaseType(db_type.lower())
self.host = host.strip()
@@ -85,6 +87,7 @@ class RDBMSConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):
self.id_column = id_column.strip() if id_column else None
self.timestamp_column = timestamp_column.strip() if timestamp_column else None
self.batch_size = batch_size
+ self.file_extension = file_extension if file_extension else ".txt"
self._connection = None
self._credentials: Dict[str, Any] = {}
@@ -450,7 +453,7 @@ class RDBMSConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):
blob=blob,
source=DocumentSource(self.db_type.value),
semantic_identifier=semantic_id,
- extension=".txt",
+ extension=self.file_extension,
doc_updated_at=doc_updated_at,
size_bytes=len(blob),
metadata=metadata if metadata else None,
diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.py
index e6efaed256..3e59a450d6 100644
--- a/rag/svr/sync_data_source.py
+++ b/rag/svr/sync_data_source.py
@@ -2192,6 +2192,7 @@ class _RDBMSBase(_CursorPersistingSyncBase):
id_column=self.conf.get("id_column") or None,
timestamp_column=self.conf.get("timestamp_column") or None,
batch_size=self.conf.get("batch_size", INDEX_BATCH_SIZE),
+ file_extension=self.conf.get("file_extension", ".txt"),
)
credentials = self.conf.get("credentials")
diff --git a/test/unit_test/rag/test_sync_data_source.py b/test/unit_test/rag/test_sync_data_source.py
index d08210cb4c..d4e7c89ef1 100644
--- a/test/unit_test/rag/test_sync_data_source.py
+++ b/test/unit_test/rag/test_sync_data_source.py
@@ -276,6 +276,7 @@ class _FakeRDBMSConnector:
id_column=None,
timestamp_column=None,
batch_size=2,
+ file_extension=None,
):
self.db_type = db_type
self.host = host
@@ -287,6 +288,7 @@ class _FakeRDBMSConnector:
self.id_column = id_column
self.timestamp_column = timestamp_column
self.batch_size = batch_size
+ self.file_extension = file_extension if file_extension else ".txt"
self.load_from_state_called = False
self.retrieve_all_slim_docs_perm_sync_called = False
self.prepare_sync_state_called = False
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index 55ee1cb6bc..d10501d1e8 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -1775,6 +1775,8 @@ Example: Virtual Hosted Style`,
'Column to use as unique document ID. If not specified, a hash of the content will be used.',
mysqlTimestampColumnTip:
'Datetime/timestamp column for incremental sync. Only rows modified after the last sync will be fetched.',
+ mysqlFileExtensionTip:
+ 'The extension used for documents synced from this source (default: .txt). Since each row is saved as one file, when set to .html or .md, select only content columns that are purely HTML or Markdown.',
postgresqlDescription:
'Connect to PostgreSQL database to sync data from tables using SQL queries.',
postgresqlQueryTip:
@@ -1787,6 +1789,8 @@ Example: Virtual Hosted Style`,
'Column to use as unique document ID. If not specified, a hash of the content will be used.',
postgresqlTimestampColumnTip:
'Datetime/timestamp column for incremental sync. Only rows modified after the last sync will be fetched.',
+ postgresqlFileExtensionTip:
+ 'The extension used for documents synced from this source (default: .txt). Since each row is saved as one file, when set to .html or .md, select only content columns that are purely HTML or Markdown.',
bigqueryDescription:
'Connect to Google BigQuery to sync rows from a table or a custom GoogleSQL query.',
bigqueryProjectIdTip:
diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsx
index eb33a360d3..138aa6e425 100644
--- a/web/src/pages/user-setting/data-source/constant/index.tsx
+++ b/web/src/pages/user-setting/data-source/constant/index.tsx
@@ -1560,6 +1560,14 @@ const generateDataSourceFormFields = (t: TFunction) => ({
placeholder: 'updated_at',
tooltip: t('setting.mysqlTimestampColumnTip'),
},
+ {
+ label: 'File Extension',
+ name: 'config.file_extension',
+ type: FormFieldType.Text,
+ required: false,
+ placeholder: '.txt',
+ tooltip: t('setting.mysqlFileExtensionTip'),
+ },
],
[DataSourceKey.POSTGRESQL]: [
{
@@ -1634,6 +1642,14 @@ const generateDataSourceFormFields = (t: TFunction) => ({
placeholder: 'updated_at',
tooltip: t('setting.postgresqlTimestampColumnTip'),
},
+ {
+ label: 'File Extension',
+ name: 'config.file_extension',
+ type: FormFieldType.Text,
+ required: false,
+ placeholder: '.txt',
+ tooltip: t('setting.postgresqlFileExtensionTip'),
+ },
],
[DataSourceKey.BIGQUERY]: [
{