RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: record generated wiki page versions (#17931)
buua4368379165c12ae
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex dc9c6f939a..9c9f9961f2 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -3174,7 +3174,7 @@ async def update_wiki_page(content_before = ""try:res = settings.docStoreConn.search(- select_fields=["id", "content_with_weight"],+ select_fields=["id", "md_with_weight", "content_with_weight"],highlight_fields=[],condition={"compile_kwd": [WIKI_PAGE_COMPILE_KWD],@@ -3190,11 +3190,11 @@ async def update_wiki_page()field_map = settings.docStoreConn.get_fields(res,- ["id", "content_with_weight"],+ ["id", "md_with_weight", "content_with_weight"],)if field_map:row_id, row = next(iter(field_map.items()))- content_before = row.get("content_with_weight") or ""+ content_before = row.get("md_with_weight") or row.get("content_with_weight") or ""except Exception:logging.exception("update_wiki_page: lookup failed for kb=%s slug=%s",@@ -3215,6 +3215,7 @@ async def update_wiki_page(ok = settings.docStoreConn.update({"id": row_id},{+ "md_with_weight": rendered,"content_with_weight": rendered,"summary_with_weight": summary,"outlinks_kwd": list(outlinks),@@ -3233,6 +3234,17 @@ async def update_wiki_page(if not ok:return True, None+ refresh_idx = getattr(settings.docStoreConn, "refresh_idx", None)+ if callable(refresh_idx):+ try:+ await thread_pool_exec(refresh_idx, index_nm)+ except Exception:+ logging.exception(+ "update_wiki_page: index refresh failed for kb=%s slug=%s",+ dataset_id,+ full_slug,+ )+# Record a file_commit row on every real change. ``record_page_edit``# returns None for empty-diff saves, which we silently swallow.try:@@ -3824,4 +3836,18 @@ async def clear_wiki(dataset_id: str, tenant_id: str):)deleted[kwd] = False+ from api.db.services.file_commit_service import FileCommitService++ if all(result is not False for result in deleted.values()):+ try:+ deleted["file_commit_history"] = FileCommitService.delete_all_page_history(dataset_id)+ except Exception:+ logging.exception(+ "clear_wiki: failed to delete page version history for kb=%s",+ dataset_id,+ )+ deleted["file_commit_history"] = False+ else:+ deleted["file_commit_history"] = False+return True, {"deleted": deleted}diff --git a/api/db/services/file_commit_service.py b/api/db/services/file_commit_service.pyindex 86117596f0..30a6df25ac 100644--- a/api/db/services/file_commit_service.py+++ b/api/db/services/file_commit_service.py@@ -801,6 +801,49 @@ class FileCommitService(CommonService):return commit_id+ @classmethod+ @DB.connection_context()+ def delete_page_history(cls, kb_id: str, page_type: str, slug: str) -> int:+ """Delete all stored versions for one Wiki page.++ Wiki versions are represented by a ``FileCommit`` plus its single+ ``FileCommitItem``. They are not workspace commits, so removing the+ page must remove both rows instead of leaving an orphaned history+ that can reappear when the same slug is generated again.+ """+ file_id = _wiki_file_id(kb_id, slug)+ commit_ids = [+ row.commit_id+ for row in FileCommitItem.select(FileCommitItem.commit_id).where((FileCommitItem.file_id == file_id) & (FileCommitItem.slug_kwd == slug) & (FileCommitItem.page_type_kwd == page_type))+ ]+ if not commit_ids:+ return 0++ with DB.atomic():+ FileCommitItem.delete().where(FileCommitItem.commit_id.in_(commit_ids)).execute()+ deleted = FileCommit.delete().where((FileCommit.folder_id == kb_id) & FileCommit.id.in_(commit_ids)).execute()+ return deleted++ @classmethod+ @DB.connection_context()+ def delete_all_page_history(cls, kb_id: str) -> int:+ """Delete all Wiki page versions belonging to a knowledge base."""+ commit_ids = [+ row.commit_id+ for row in (+ FileCommitItem.select(FileCommitItem.commit_id)+ .join(FileCommit, on=(FileCommit.id == FileCommitItem.commit_id))+ .where((FileCommit.folder_id == kb_id) & FileCommitItem.slug_kwd.is_null(False) & FileCommitItem.page_type_kwd.is_null(False))+ )+ ]+ if not commit_ids:+ return 0++ with DB.atomic():+ FileCommitItem.delete().where(FileCommitItem.commit_id.in_(commit_ids)).execute()+ deleted = FileCommit.delete().where((FileCommit.folder_id == kb_id) & FileCommit.id.in_(commit_ids)).execute()+ return deleted+@classmethod@DB.connection_context()def list_page_commits(diff --git a/rag/advanced_rag/knowlege_compile/wiki_incremental.py b/rag/advanced_rag/knowlege_compile/wiki_incremental.pyindex b9f5f727a9..b8e79ce875 100644--- a/rag/advanced_rag/knowlege_compile/wiki_incremental.py+++ b/rag/advanced_rag/knowlege_compile/wiki_incremental.py@@ -1567,12 +1567,19 @@ async def _wiki_refine_page(return existing_pageif mode == "delete":- await thread_pool_exec(+ deleted_count = await thread_pool_exec(settings.docStoreConn.delete,{"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "slug_kwd": [page_id]},search.index_name(tenant_id),kb_id,)+ if not isinstance(deleted_count, int) or deleted_count <= 0:+ logging.warning("wiki: page deletion did not remove page=%s", page_id)+ return existing_page+ from api.db.services.file_commit_service import FileCommitService++ commit_slug = page_id if page_id.startswith(f"{page_type_kwd}/") else f"{page_type_kwd}/{page_id}"+ FileCommitService.delete_page_history(kb_id, page_type_kwd, commit_slug)return None# WeKnora-style verbatim evidence: load the ACTUAL source-chunk text for@@ -1732,6 +1739,31 @@ async def _wiki_refine_page(kb_id,)+ # Keep generated pages in the same version history as manual edits. The+ # incremental compiler is the normal Wiki path, so without this record the+ # first generated page (and subsequent generated revisions) are invisible+ # to the commits API.+ from api.db.services.file_commit_service import FileCommitService++ content_before = ""+ if existing_page:+ content_before = existing_page.get("md_with_weight") or existing_page.get("content_with_weight") or ""+ commit_slug = page_id if page_id.startswith(f"{page_type_kwd}/") else f"{page_type_kwd}/{page_id}"+ try:+ FileCommitService.record_page_edit(+ tenant_id=tenant_id,+ kb_id=kb_id,+ page_type=page_type_kwd,+ slug=commit_slug,+ content_before=content_before,+ content_after=content,+ title="Regenerated by artifact compilation",+ comments=f"Auto-update via incremental wiki compilation (action={mode.upper()})",+ user_id=None,+ )+ except Exception:+ logging.exception("wiki: generated page version record failed for page=%s", page_id)+return pagediff --git a/rag/svr/task_executor_refactor/dataset_wiki_generator.py b/rag/svr/task_executor_refactor/dataset_wiki_generator.pyindex 3cf929c606..d46960905a 100644--- a/rag/svr/task_executor_refactor/dataset_wiki_generator.py+++ b/rag/svr/task_executor_refactor/dataset_wiki_generator.py@@ -402,9 +402,11 @@ async def _wiki_delete_deleted_doc_state(deleted = set(deleted_doc_ids)derived_kwds = list(WIKI_DERIVED_COMPILE_KWDS)- select_fields = ["id", "source_doc_ids"]+ select_fields = ["id", "source_doc_ids", "compile_kwd", "slug_kwd", "page_type_kwd"]to_delete: list[str] = []to_shrink: list[tuple[str, list[str]]] = []+ page_history_to_delete: list[tuple[str, str, str]] = []+ failed_delete_row_ids: set[str] = set()offset = 0page_size = 1000while True:@@ -440,21 +442,44 @@ async def _wiki_delete_deleted_doc_state(to_shrink.append((row_id, remaining))else:to_delete.append(row_id)+ if row.get("compile_kwd") == WIKI_PAGE_COMPILE_KWD:+ slug = row.get("slug_kwd")+ if isinstance(slug, str) and slug:+ page_history_to_delete.append((row_id, slug, row.get("page_type_kwd") or "concept"))if len(field_map) < page_size:breakoffset += page_size# Drop rows with no surviving owner (delete by id in batches).for i in range(0, len(to_delete), page_size):+ batch_ids = to_delete[i : i + page_size]try:- await thread_pool_exec(+ deleted_count = await thread_pool_exec(settings.docStoreConn.delete,- {"id": to_delete[i : i + page_size]},+ {"id": batch_ids},index,kb_id,)+ if not isinstance(deleted_count, int) or deleted_count != len(batch_ids):+ failed_delete_row_ids.update(batch_ids)except Exception:logging.exception("wiki: failed to drop orphaned derived rows in kb=%s", kb_id)+ failed_delete_row_ids.update(batch_ids)++ if page_history_to_delete:+ from api.db.services.file_commit_service import FileCommitService++ for row_id, slug, page_type in page_history_to_delete:+ if row_id in failed_delete_row_ids:+ continue+ try:+ FileCommitService.delete_page_history(kb_id, page_type, slug)+ except Exception:+ logging.exception(+ "wiki: failed to delete version history for removed page=%s kb=%s",+ slug,+ kb_id,+ )# Shrink rows still owned by surviving docs to just those docs.for row_id, remaining in to_shrink:@@ -733,15 +758,17 @@ async def persist_wiki_pages(# Capture the prior rendered content for every slug we're about to# overwrite, so the per-page commit row downstream has a real diff- # baseline. Single batch read by slug_kwd IN [...] — one round-trip- # regardless of page count. Failures here degrade gracefully.+ # baseline. Prefer md_with_weight because that is the canonical field+ # consumed by the Wiki page API, with content_with_weight as fallback.+ # Single batch read by slug_kwd IN [...] — one round-trip regardless of+ # page count. Failures here degrade gracefully.target_slugs: list[str] = [(p.get("slug") or "").strip() for p in pages if isinstance(p.get("slug"), str) and p.get("slug")]prior_by_slug: dict[str, str] = {}if target_slugs:try:res = await thread_pool_exec(settings.docStoreConn.search,- ["id", "slug_kwd", "content_with_weight"],+ ["id", "slug_kwd", "md_with_weight", "content_with_weight"],[],{"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "slug_kwd": list(target_slugs)},[],@@ -753,11 +780,11 @@ async def persist_wiki_pages()field_map = settings.docStoreConn.get_fields(res,- ["id", "slug_kwd", "content_with_weight"],+ ["id", "slug_kwd", "md_with_weight", "content_with_weight"],)for row in (field_map or {}).values():s = row.get("slug_kwd")- c = row.get("content_with_weight")+ c = row.get("md_with_weight") or row.get("content_with_weight")if isinstance(s, str) and isinstance(c, str):prior_by_slug[s] = cexcept Exception:@@ -838,6 +865,7 @@ async def persist_wiki_pages("related_kb_pages_kwd": list(page.get("related_kb_pages") or []),"source_chunk_ids": list(page.get("source_chunk_ids") or []),"source_doc_ids": list(page.get("source_doc_ids") or []),+ "md_with_weight": content_md,"content_with_weight": content_md,# Summary kept verbatim alongside the rendered body so the# viewer can render it as a distinct (smaller) block above@@ -862,7 +890,7 @@ async def persist_wiki_pages(returntry:- await thread_pool_exec(settings.docStoreConn.insert, rows, index, ctx.kb_id)+ insert_errors = await thread_pool_exec(settings.docStoreConn.insert, rows, index, ctx.kb_id)except Exception:logging.exception("wiki_persist: bulk insert failed for kb=%s (rows=%d)",@@ -871,6 +899,38 @@ async def persist_wiki_pages()return+ if insert_errors:+ logging.warning(+ "wiki_persist: page insert returned %d error(s) for kb=%s",+ len(insert_errors),+ )++ # Verify the rows that are actually visible after the bulk operation. Some+ # backends can report partial bulk failures without raising, so recording a+ # version for every input page would create history for pages that were not+ # persisted.+ try:+ persisted_res = await thread_pool_exec(+ settings.docStoreConn.search,+ ["id", "slug_kwd"],+ [],+ {"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "id": [row["id"] for row in rows]},+ [],+ OrderByExpr(),+ 0,+ len(rows),+ index,+ [ctx.kb_id],+ )+ persisted_fields = settings.docStoreConn.get_fields(persisted_res, ["id", "slug_kwd"])+ persisted_page_slugs = {row.get("slug_kwd") for row in (persisted_fields or {}).values() if isinstance(row.get("slug_kwd"), str)}+ except Exception:+ logging.exception(+ "wiki_persist: failed to verify persisted pages for kb=%s; skipping history records",+ kb_id_str,+ )+ return+if topics_by_name:try:await _ensure_wiki_topic_rows(ctx, index, kb_id_str, topics_by_name, topic_doc_ids)@@ -886,9 +946,7 @@ async def persist_wiki_pages(# compile.for page in pages:slug = (page.get("slug") or "").strip()- if not slug:- continue- if not prior_by_slug.get(slug, ""):+ if not slug or slug not in persisted_page_slugs:continuecontent_md = page.get("content_md_rendered") or page.get("content_md") or page.get("content_md_raw") or ""action = (page.get("action") or "CREATE").upper()diff --git a/rag/utils/es_conn.py b/rag/utils/es_conn.pyindex 4c8bfc24a6..b06a2c3e5c 100644--- a/rag/utils/es_conn.py+++ b/rag/utils/es_conn.py@@ -102,6 +102,10 @@ class ESConnection(ESConnectionBase):CRUD operations"""+ def refresh_idx(self, index_name: str) -> bool:+ self.es.indices.refresh(index=index_name)+ return True+def _es_search_once(self, index_names: list[str], query: dict, track_total_hits: bool):return self.es.search(index=index_names, body=query, timeout="600s", track_total_hits=track_total_hits)diff --git a/web/src/constants/knowledge.ts b/web/src/constants/knowledge.tsindex 55e4fb1407..b97f5ef579 100644--- a/web/src/constants/knowledge.ts+++ b/web/src/constants/knowledge.ts@@ -120,7 +120,7 @@ export enum ParseType {export enum ProcessingType {knowledgeGraph = 'Graph',raptor = 'RAPTOR',- artifact = 'Artifact',+ artifact = 'wiki',skill = 'Skill',mindmap = 'Mindmap',timeline = 'Timeline',diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.tsindex d33797d88f..739ec631a6 100644--- a/web/src/hooks/use-knowledge-request.ts+++ b/web/src/hooks/use-knowledge-request.ts@@ -727,13 +727,15 @@ export const useUpdateArtifactPage = () => {);if (data.code === 0) {message.success(i18n.t(`message.updated`));- queryClient.invalidateQueries({- queryKey: ArtifactKeys.detail(- knowledgeBaseId,- params.pageType,- params.slug,- ),- });+ const detailKey = ArtifactKeys.detail(+ knowledgeBaseId,+ params.pageType,+ params.slug,+ );+ if (data.data) {+ queryClient.setQueryData(detailKey, data.data);+ }+ await queryClient.invalidateQueries({ queryKey: detailKey });}return data;},@@ -972,7 +974,7 @@ export const useRunArtifactIndex = (kind: string) => {if (isGoDatasetBackend()) {throw new Error(i18n.t('message.compileNotSupported'));}- const { data } = await runIndex(knowledgeBaseId, 'artifact');+ const { data } = await runIndex(knowledgeBaseId, 'wiki');if (data?.code === 0) {message.success(i18n.t('message.operated'));queryClient.invalidateQueries({