RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Custom the file extension when saving DB fields, and Parse HTML fragments not just full HTML doc (#15998)
Ren Wei (任卫)c9a6f92738bd
diff --git a/common/data_source/rdbms_connector.py b/common/data_source/rdbms_connector.pyindex 4614d95875..c01bc1a11e 100644--- a/common/data_source/rdbms_connector.py+++ b/common/data_source/rdbms_connector.py@@ -57,6 +57,7 @@ class RDBMSConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):id_column: Optional[str] = None,timestamp_column: Optional[str] = None,batch_size: int = INDEX_BATCH_SIZE,+ file_extension=None,) -> None:"""Initialize the RDBMS connector.@@ -72,6 +73,7 @@ class RDBMSConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):id_column: Column to use as unique document ID (optional, will generate hash if not provided)timestamp_column: Column to use for incremental sync (optional, must be datetime/timestamp type)batch_size: Number of documents per batch+ file_extension: File Extension to save content column into"""self.db_type = DatabaseType(db_type.lower())self.host = host.strip()@@ -85,6 +87,7 @@ class RDBMSConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):self.id_column = id_column.strip() if id_column else Noneself.timestamp_column = timestamp_column.strip() if timestamp_column else Noneself.batch_size = batch_size+ self.file_extension = file_extension if file_extension else ".txt"self._connection = Noneself._credentials: Dict[str, Any] = {}@@ -450,7 +453,7 @@ class RDBMSConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):blob=blob,source=DocumentSource(self.db_type.value),semantic_identifier=semantic_id,- extension=".txt",+ extension=self.file_extension,doc_updated_at=doc_updated_at,size_bytes=len(blob),metadata=metadata if metadata else None,diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.pyindex e6efaed256..3e59a450d6 100644--- a/rag/svr/sync_data_source.py+++ b/rag/svr/sync_data_source.py@@ -2192,6 +2192,7 @@ class _RDBMSBase(_CursorPersistingSyncBase):id_column=self.conf.get("id_column") or None,timestamp_column=self.conf.get("timestamp_column") or None,batch_size=self.conf.get("batch_size", INDEX_BATCH_SIZE),+ file_extension=self.conf.get("file_extension", ".txt"),)credentials = self.conf.get("credentials")diff --git a/test/unit_test/rag/test_sync_data_source.py b/test/unit_test/rag/test_sync_data_source.pyindex d08210cb4c..d4e7c89ef1 100644--- a/test/unit_test/rag/test_sync_data_source.py+++ b/test/unit_test/rag/test_sync_data_source.py@@ -276,6 +276,7 @@ class _FakeRDBMSConnector:id_column=None,timestamp_column=None,batch_size=2,+ file_extension=None,):self.db_type = db_typeself.host = host@@ -287,6 +288,7 @@ class _FakeRDBMSConnector:self.id_column = id_columnself.timestamp_column = timestamp_columnself.batch_size = batch_size+ self.file_extension = file_extension if file_extension else ".txt"self.load_from_state_called = Falseself.retrieve_all_slim_docs_perm_sync_called = Falseself.prepare_sync_state_called = Falsediff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex 55ee1cb6bc..d10501d1e8 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -1775,6 +1775,8 @@ Example: Virtual Hosted Style`,'Column to use as unique document ID. If not specified, a hash of the content will be used.',mysqlTimestampColumnTip:'Datetime/timestamp column for incremental sync. Only rows modified after the last sync will be fetched.',+ mysqlFileExtensionTip:+ 'The extension used for documents synced from this source (default: .txt). Since each row is saved as one file, when set to .html or .md, select only content columns that are purely HTML or Markdown.',postgresqlDescription:'Connect to PostgreSQL database to sync data from tables using SQL queries.',postgresqlQueryTip:@@ -1787,6 +1789,8 @@ Example: Virtual Hosted Style`,'Column to use as unique document ID. If not specified, a hash of the content will be used.',postgresqlTimestampColumnTip:'Datetime/timestamp column for incremental sync. Only rows modified after the last sync will be fetched.',+ postgresqlFileExtensionTip:+ 'The extension used for documents synced from this source (default: .txt). Since each row is saved as one file, when set to .html or .md, select only content columns that are purely HTML or Markdown.',bigqueryDescription:'Connect to Google BigQuery to sync rows from a table or a custom GoogleSQL query.',bigqueryProjectIdTip:diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsxindex eb33a360d3..138aa6e425 100644--- a/web/src/pages/user-setting/data-source/constant/index.tsx+++ b/web/src/pages/user-setting/data-source/constant/index.tsx@@ -1560,6 +1560,14 @@ const generateDataSourceFormFields = (t: TFunction) => ({placeholder: 'updated_at',tooltip: t('setting.mysqlTimestampColumnTip'),},+ {+ label: 'File Extension',+ name: 'config.file_extension',+ type: FormFieldType.Text,+ required: false,+ placeholder: '.txt',+ tooltip: t('setting.mysqlFileExtensionTip'),+ },],[DataSourceKey.POSTGRESQL]: [{@@ -1634,6 +1642,14 @@ const generateDataSourceFormFields = (t: TFunction) => ({placeholder: 'updated_at',tooltip: t('setting.postgresqlTimestampColumnTip'),},+ {+ label: 'File Extension',+ name: 'config.file_extension',+ type: FormFieldType.Text,+ required: false,+ placeholder: '.txt',+ tooltip: t('setting.postgresqlFileExtensionTip'),+ },],[DataSourceKey.BIGQUERY]: [{