Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: record generated wiki page versions (#17931)

buua4368379165c12ae
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index dc9c6f939a..9c9f9961f2 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -3174,7 +3174,7 @@ async def update_wiki_page(
content_before = ""
try:
res = settings.docStoreConn.search(
- select_fields=["id", "content_with_weight"],
+ select_fields=["id", "md_with_weight", "content_with_weight"],
highlight_fields=[],
condition={
"compile_kwd": [WIKI_PAGE_COMPILE_KWD],
@@ -3190,11 +3190,11 @@ async def update_wiki_page(
)
field_map = settings.docStoreConn.get_fields(
res,
- ["id", "content_with_weight"],
+ ["id", "md_with_weight", "content_with_weight"],
)
if field_map:
row_id, row = next(iter(field_map.items()))
- content_before = row.get("content_with_weight") or ""
+ content_before = row.get("md_with_weight") or row.get("content_with_weight") or ""
except Exception:
logging.exception(
"update_wiki_page: lookup failed for kb=%s slug=%s",
@@ -3215,6 +3215,7 @@ async def update_wiki_page(
ok = settings.docStoreConn.update(
{"id": row_id},
{
+ "md_with_weight": rendered,
"content_with_weight": rendered,
"summary_with_weight": summary,
"outlinks_kwd": list(outlinks),
@@ -3233,6 +3234,17 @@ async def update_wiki_page(
if not ok:
return True, None
+ refresh_idx = getattr(settings.docStoreConn, "refresh_idx", None)
+ if callable(refresh_idx):
+ try:
+ await thread_pool_exec(refresh_idx, index_nm)
+ except Exception:
+ logging.exception(
+ "update_wiki_page: index refresh failed for kb=%s slug=%s",
+ dataset_id,
+ full_slug,
+ )
+
# Record a file_commit row on every real change. ``record_page_edit``
# returns None for empty-diff saves, which we silently swallow.
try:
@@ -3824,4 +3836,18 @@ async def clear_wiki(dataset_id: str, tenant_id: str):
)
deleted[kwd] = False
+ from api.db.services.file_commit_service import FileCommitService
+
+ if all(result is not False for result in deleted.values()):
+ try:
+ deleted["file_commit_history"] = FileCommitService.delete_all_page_history(dataset_id)
+ except Exception:
+ logging.exception(
+ "clear_wiki: failed to delete page version history for kb=%s",
+ dataset_id,
+ )
+ deleted["file_commit_history"] = False
+ else:
+ deleted["file_commit_history"] = False
+
return True, {"deleted": deleted}
diff --git a/api/db/services/file_commit_service.py b/api/db/services/file_commit_service.py
index 86117596f0..30a6df25ac 100644
--- a/api/db/services/file_commit_service.py
+++ b/api/db/services/file_commit_service.py
@@ -801,6 +801,49 @@ class FileCommitService(CommonService):
return commit_id
+ @classmethod
+ @DB.connection_context()
+ def delete_page_history(cls, kb_id: str, page_type: str, slug: str) -> int:
+ """Delete all stored versions for one Wiki page.
+
+ Wiki versions are represented by a ``FileCommit`` plus its single
+ ``FileCommitItem``. They are not workspace commits, so removing the
+ page must remove both rows instead of leaving an orphaned history
+ that can reappear when the same slug is generated again.
+ """
+ file_id = _wiki_file_id(kb_id, slug)
+ commit_ids = [
+ row.commit_id
+ for row in FileCommitItem.select(FileCommitItem.commit_id).where((FileCommitItem.file_id == file_id) & (FileCommitItem.slug_kwd == slug) & (FileCommitItem.page_type_kwd == page_type))
+ ]
+ if not commit_ids:
+ return 0
+
+ with DB.atomic():
+ FileCommitItem.delete().where(FileCommitItem.commit_id.in_(commit_ids)).execute()
+ deleted = FileCommit.delete().where((FileCommit.folder_id == kb_id) & FileCommit.id.in_(commit_ids)).execute()
+ return deleted
+
+ @classmethod
+ @DB.connection_context()
+ def delete_all_page_history(cls, kb_id: str) -> int:
+ """Delete all Wiki page versions belonging to a knowledge base."""
+ commit_ids = [
+ row.commit_id
+ for row in (
+ FileCommitItem.select(FileCommitItem.commit_id)
+ .join(FileCommit, on=(FileCommit.id == FileCommitItem.commit_id))
+ .where((FileCommit.folder_id == kb_id) & FileCommitItem.slug_kwd.is_null(False) & FileCommitItem.page_type_kwd.is_null(False))
+ )
+ ]
+ if not commit_ids:
+ return 0
+
+ with DB.atomic():
+ FileCommitItem.delete().where(FileCommitItem.commit_id.in_(commit_ids)).execute()
+ deleted = FileCommit.delete().where((FileCommit.folder_id == kb_id) & FileCommit.id.in_(commit_ids)).execute()
+ return deleted
+
@classmethod
@DB.connection_context()
def list_page_commits(
diff --git a/rag/advanced_rag/knowlege_compile/wiki_incremental.py b/rag/advanced_rag/knowlege_compile/wiki_incremental.py
index b9f5f727a9..b8e79ce875 100644
--- a/rag/advanced_rag/knowlege_compile/wiki_incremental.py
+++ b/rag/advanced_rag/knowlege_compile/wiki_incremental.py
@@ -1567,12 +1567,19 @@ async def _wiki_refine_page(
return existing_page
if mode == "delete":
- await thread_pool_exec(
+ deleted_count = await thread_pool_exec(
settings.docStoreConn.delete,
{"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "slug_kwd": [page_id]},
search.index_name(tenant_id),
kb_id,
)
+ if not isinstance(deleted_count, int) or deleted_count <= 0:
+ logging.warning("wiki: page deletion did not remove page=%s", page_id)
+ return existing_page
+ from api.db.services.file_commit_service import FileCommitService
+
+ commit_slug = page_id if page_id.startswith(f"{page_type_kwd}/") else f"{page_type_kwd}/{page_id}"
+ FileCommitService.delete_page_history(kb_id, page_type_kwd, commit_slug)
return None
# WeKnora-style verbatim evidence: load the ACTUAL source-chunk text for
@@ -1732,6 +1739,31 @@ async def _wiki_refine_page(
kb_id,
)
+ # Keep generated pages in the same version history as manual edits. The
+ # incremental compiler is the normal Wiki path, so without this record the
+ # first generated page (and subsequent generated revisions) are invisible
+ # to the commits API.
+ from api.db.services.file_commit_service import FileCommitService
+
+ content_before = ""
+ if existing_page:
+ content_before = existing_page.get("md_with_weight") or existing_page.get("content_with_weight") or ""
+ commit_slug = page_id if page_id.startswith(f"{page_type_kwd}/") else f"{page_type_kwd}/{page_id}"
+ try:
+ FileCommitService.record_page_edit(
+ tenant_id=tenant_id,
+ kb_id=kb_id,
+ page_type=page_type_kwd,
+ slug=commit_slug,
+ content_before=content_before,
+ content_after=content,
+ title="Regenerated by artifact compilation",
+ comments=f"Auto-update via incremental wiki compilation (action={mode.upper()})",
+ user_id=None,
+ )
+ except Exception:
+ logging.exception("wiki: generated page version record failed for page=%s", page_id)
+
return page
diff --git a/rag/svr/task_executor_refactor/dataset_wiki_generator.py b/rag/svr/task_executor_refactor/dataset_wiki_generator.py
index 3cf929c606..d46960905a 100644
--- a/rag/svr/task_executor_refactor/dataset_wiki_generator.py
+++ b/rag/svr/task_executor_refactor/dataset_wiki_generator.py
@@ -402,9 +402,11 @@ async def _wiki_delete_deleted_doc_state(
deleted = set(deleted_doc_ids)
derived_kwds = list(WIKI_DERIVED_COMPILE_KWDS)
- select_fields = ["id", "source_doc_ids"]
+ select_fields = ["id", "source_doc_ids", "compile_kwd", "slug_kwd", "page_type_kwd"]
to_delete: list[str] = []
to_shrink: list[tuple[str, list[str]]] = []
+ page_history_to_delete: list[tuple[str, str, str]] = []
+ failed_delete_row_ids: set[str] = set()
offset = 0
page_size = 1000
while True:
@@ -440,21 +442,44 @@ async def _wiki_delete_deleted_doc_state(
to_shrink.append((row_id, remaining))
else:
to_delete.append(row_id)
+ if row.get("compile_kwd") == WIKI_PAGE_COMPILE_KWD:
+ slug = row.get("slug_kwd")
+ if isinstance(slug, str) and slug:
+ page_history_to_delete.append((row_id, slug, row.get("page_type_kwd") or "concept"))
if len(field_map) < page_size:
break
offset += page_size
# Drop rows with no surviving owner (delete by id in batches).
for i in range(0, len(to_delete), page_size):
+ batch_ids = to_delete[i : i + page_size]
try:
- await thread_pool_exec(
+ deleted_count = await thread_pool_exec(
settings.docStoreConn.delete,
- {"id": to_delete[i : i + page_size]},
+ {"id": batch_ids},
index,
kb_id,
)
+ if not isinstance(deleted_count, int) or deleted_count != len(batch_ids):
+ failed_delete_row_ids.update(batch_ids)
except Exception:
logging.exception("wiki: failed to drop orphaned derived rows in kb=%s", kb_id)
+ failed_delete_row_ids.update(batch_ids)
+
+ if page_history_to_delete:
+ from api.db.services.file_commit_service import FileCommitService
+
+ for row_id, slug, page_type in page_history_to_delete:
+ if row_id in failed_delete_row_ids:
+ continue
+ try:
+ FileCommitService.delete_page_history(kb_id, page_type, slug)
+ except Exception:
+ logging.exception(
+ "wiki: failed to delete version history for removed page=%s kb=%s",
+ slug,
+ kb_id,
+ )
# Shrink rows still owned by surviving docs to just those docs.
for row_id, remaining in to_shrink:
@@ -733,15 +758,17 @@ async def persist_wiki_pages(
# Capture the prior rendered content for every slug we're about to
# overwrite, so the per-page commit row downstream has a real diff
- # baseline. Single batch read by slug_kwd IN [...] — one round-trip
- # regardless of page count. Failures here degrade gracefully.
+ # baseline. Prefer md_with_weight because that is the canonical field
+ # consumed by the Wiki page API, with content_with_weight as fallback.
+ # Single batch read by slug_kwd IN [...] — one round-trip regardless of
+ # page count. Failures here degrade gracefully.
target_slugs: list[str] = [(p.get("slug") or "").strip() for p in pages if isinstance(p.get("slug"), str) and p.get("slug")]
prior_by_slug: dict[str, str] = {}
if target_slugs:
try:
res = await thread_pool_exec(
settings.docStoreConn.search,
- ["id", "slug_kwd", "content_with_weight"],
+ ["id", "slug_kwd", "md_with_weight", "content_with_weight"],
[],
{"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "slug_kwd": list(target_slugs)},
[],
@@ -753,11 +780,11 @@ async def persist_wiki_pages(
)
field_map = settings.docStoreConn.get_fields(
res,
- ["id", "slug_kwd", "content_with_weight"],
+ ["id", "slug_kwd", "md_with_weight", "content_with_weight"],
)
for row in (field_map or {}).values():
s = row.get("slug_kwd")
- c = row.get("content_with_weight")
+ c = row.get("md_with_weight") or row.get("content_with_weight")
if isinstance(s, str) and isinstance(c, str):
prior_by_slug[s] = c
except Exception:
@@ -838,6 +865,7 @@ async def persist_wiki_pages(
"related_kb_pages_kwd": list(page.get("related_kb_pages") or []),
"source_chunk_ids": list(page.get("source_chunk_ids") or []),
"source_doc_ids": list(page.get("source_doc_ids") or []),
+ "md_with_weight": content_md,
"content_with_weight": content_md,
# Summary kept verbatim alongside the rendered body so the
# viewer can render it as a distinct (smaller) block above
@@ -862,7 +890,7 @@ async def persist_wiki_pages(
return
try:
- await thread_pool_exec(settings.docStoreConn.insert, rows, index, ctx.kb_id)
+ insert_errors = await thread_pool_exec(settings.docStoreConn.insert, rows, index, ctx.kb_id)
except Exception:
logging.exception(
"wiki_persist: bulk insert failed for kb=%s (rows=%d)",
@@ -871,6 +899,38 @@ async def persist_wiki_pages(
)
return
+ if insert_errors:
+ logging.warning(
+ "wiki_persist: page insert returned %d error(s) for kb=%s",
+ len(insert_errors),
+ )
+
+ # Verify the rows that are actually visible after the bulk operation. Some
+ # backends can report partial bulk failures without raising, so recording a
+ # version for every input page would create history for pages that were not
+ # persisted.
+ try:
+ persisted_res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ ["id", "slug_kwd"],
+ [],
+ {"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "id": [row["id"] for row in rows]},
+ [],
+ OrderByExpr(),
+ 0,
+ len(rows),
+ index,
+ [ctx.kb_id],
+ )
+ persisted_fields = settings.docStoreConn.get_fields(persisted_res, ["id", "slug_kwd"])
+ persisted_page_slugs = {row.get("slug_kwd") for row in (persisted_fields or {}).values() if isinstance(row.get("slug_kwd"), str)}
+ except Exception:
+ logging.exception(
+ "wiki_persist: failed to verify persisted pages for kb=%s; skipping history records",
+ kb_id_str,
+ )
+ return
+
if topics_by_name:
try:
await _ensure_wiki_topic_rows(ctx, index, kb_id_str, topics_by_name, topic_doc_ids)
@@ -886,9 +946,7 @@ async def persist_wiki_pages(
# compile.
for page in pages:
slug = (page.get("slug") or "").strip()
- if not slug:
- continue
- if not prior_by_slug.get(slug, ""):
+ if not slug or slug not in persisted_page_slugs:
continue
content_md = page.get("content_md_rendered") or page.get("content_md") or page.get("content_md_raw") or ""
action = (page.get("action") or "CREATE").upper()
diff --git a/rag/utils/es_conn.py b/rag/utils/es_conn.py
index 4c8bfc24a6..b06a2c3e5c 100644
--- a/rag/utils/es_conn.py
+++ b/rag/utils/es_conn.py
@@ -102,6 +102,10 @@ class ESConnection(ESConnectionBase):
CRUD operations
"""
+ def refresh_idx(self, index_name: str) -> bool:
+ self.es.indices.refresh(index=index_name)
+ return True
+
def _es_search_once(self, index_names: list[str], query: dict, track_total_hits: bool):
return self.es.search(index=index_names, body=query, timeout="600s", track_total_hits=track_total_hits)
diff --git a/web/src/constants/knowledge.ts b/web/src/constants/knowledge.ts
index 55e4fb1407..b97f5ef579 100644
--- a/web/src/constants/knowledge.ts
+++ b/web/src/constants/knowledge.ts
@@ -120,7 +120,7 @@ export enum ParseType {
export enum ProcessingType {
knowledgeGraph = 'Graph',
raptor = 'RAPTOR',
- artifact = 'Artifact',
+ artifact = 'wiki',
skill = 'Skill',
mindmap = 'Mindmap',
timeline = 'Timeline',
diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.ts
index d33797d88f..739ec631a6 100644
--- a/web/src/hooks/use-knowledge-request.ts
+++ b/web/src/hooks/use-knowledge-request.ts
@@ -727,13 +727,15 @@ export const useUpdateArtifactPage = () => {
);
if (data.code === 0) {
message.success(i18n.t(`message.updated`));
- queryClient.invalidateQueries({
- queryKey: ArtifactKeys.detail(
- knowledgeBaseId,
- params.pageType,
- params.slug,
- ),
- });
+ const detailKey = ArtifactKeys.detail(
+ knowledgeBaseId,
+ params.pageType,
+ params.slug,
+ );
+ if (data.data) {
+ queryClient.setQueryData(detailKey, data.data);
+ }
+ await queryClient.invalidateQueries({ queryKey: detailKey });
}
return data;
},
@@ -972,7 +974,7 @@ export const useRunArtifactIndex = (kind: string) => {
if (isGoDatasetBackend()) {
throw new Error(i18n.t('message.compileNotSupported'));
}
- const { data } = await runIndex(knowledgeBaseId, 'artifact');
+ const { data } = await runIndex(knowledgeBaseId, 'wiki');
if (data?.code === 0) {
message.success(i18n.t('message.operated'));
queryClient.invalidateQueries({