Skip to content

A Simple and Universal Swarm Intelligence Engine, Predicting Anything. 简洁通用的群体智能引擎,预测万物

fix: modernize Zep Cloud integration

666ghje4aa38dff249
diff --git a/.env.example b/.env.example
index 78a3b72c..10276350 100644
--- a/.env.example
+++ b/.env.example
@@ -8,9 +8,13 @@ LLM_MODEL_NAME=qwen-plus
# ===== ZEP记忆图谱配置 =====
# 每月免费额度即可支撑简单使用:https://app.getzep.com/
ZEP_API_KEY=your_zep_api_key_here
+# 单次 Zep Cloud 请求超时(秒)
+ZEP_REQUEST_TIMEOUT_SECONDS=30
+# 文档批次或模拟 episode 等待处理完成的最长时间(秒)
+ZEP_INGESTION_TIMEOUT_SECONDS=600
# ===== 加速 LLM 配置(可选)=====
# 注意如果不使用加速配置,env文件中就不要出现下面的配置项
LLM_BOOST_API_KEY=your_api_key_here
LLM_BOOST_BASE_URL=your_base_url_here
-LLM_BOOST_MODEL_NAME=your_model_name_here
\ No newline at end of file
+LLM_BOOST_MODEL_NAME=your_model_name_here
diff --git a/backend/app/api/graph.py b/backend/app/api/graph.py
index 82b8d713..cfaa2627 100644
--- a/backend/app/api/graph.py
+++ b/backend/app/api/graph.py
@@ -6,21 +6,115 @@
import os
import traceback
import threading
+from contextlib import ExitStack, nullcontext
from flask import request, jsonify
+from zep_cloud import NotFoundError
from . import graph_bp
from ..config import Config
from ..services.ontology_generator import OntologyGenerator
-from ..services.graph_builder import GraphBuilderService
+from ..services.graph_builder import BatchSubmission, GraphBuilderService
from ..services.text_processor import TextProcessor
from ..utils.file_parser import FileParser
from ..utils.logger import get_logger
from ..utils.locale import t, get_locale, set_locale
+from ..utils.zep_lifecycle import get_graph_readers, graph_lifecycle_lock
from ..models.task import TaskManager, TaskStatus
from ..models.project import ProjectManager, ProjectStatus
+from ..services.simulation_manager import SimulationManager
+from ..services.simulation_runner import SimulationRunner, RunnerStatus
+from ..services.zep_graph_memory_updater import ZepGraphMemoryManager
# 获取日志器
logger = get_logger('mirofish.api')
+_build_locks: dict[str, threading.Lock] = {}
+_build_locks_guard = threading.Lock()
+
+
+class GraphInUseError(RuntimeError):
+ pass
+
+
+def _active_graph_consumers(graph_id: str) -> list[str]:
+ active = {
+ f"report:{reader_id}"
+ for reader_id in get_graph_readers(graph_id)
+ }
+ for simulation_id in ZepGraphMemoryManager.get_simulation_ids_for_graph(graph_id):
+ finalization_lock = SimulationRunner._finalization_lock(simulation_id)
+ if not finalization_lock.acquire(blocking=False):
+ active.add(simulation_id)
+ continue
+ try:
+ run_state = SimulationRunner.get_run_state(simulation_id)
+ if run_state and run_state.runner_status == RunnerStatus.FAILED:
+ # reset/delete is the explicit recovery path for an incomplete,
+ # non-replayable write. Serialize it against a retry drain.
+ ZepGraphMemoryManager.discard_inactive_updater(simulation_id)
+ SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
+ continue
+ active.add(simulation_id)
+ finally:
+ finalization_lock.release()
+ active_runner_statuses = {
+ RunnerStatus.STARTING,
+ RunnerStatus.RUNNING,
+ RunnerStatus.PAUSED,
+ RunnerStatus.STOPPING,
+ }
+ for simulation in SimulationManager().list_simulations():
+ if simulation.graph_id != graph_id:
+ continue
+ run_state = SimulationRunner.get_run_state(simulation.simulation_id)
+ if run_state and run_state.runner_status in active_runner_statuses:
+ active.add(simulation.simulation_id)
+ return sorted(active)
+
+
+def _delete_cloud_graph_if_present(graph_id: str | None) -> None:
+ """Delete a referenced Cloud graph without retrying the mutation."""
+
+ if not graph_id:
+ return
+ # Keep the consumer check and Cloud mutation in one critical section. The
+ # callers that also clear local references hold this re-entrant lock around
+ # both operations.
+ with graph_lifecycle_lock(graph_id):
+ active_simulations = _active_graph_consumers(graph_id)
+ if active_simulations:
+ raise GraphInUseError(
+ f"Graph {graph_id} is in use by active consumer(s): "
+ f"{', '.join(active_simulations)}"
+ )
+ try:
+ GraphBuilderService(api_key=Config.ZEP_API_KEY).delete_graph(graph_id)
+ except NotFoundError:
+ logger.info("Zep Cloud graph already absent: %s", graph_id)
+
+
+def _clear_project_graph_reference(project) -> None:
+ project.graph_id = None
+ project.graph_build_task_id = None
+ project.zep_batch_id = None
+ project.zep_batch_operation_id = None
+ project.error = None
+
+
+def _project_build_lock(project_id: str) -> threading.Lock:
+ with _build_locks_guard:
+ return _build_locks.setdefault(project_id, threading.Lock())
+
+
+def _project_has_active_build(project) -> bool:
+ if project.status != ProjectStatus.GRAPH_BUILDING:
+ return False
+ if not project.graph_build_task_id:
+ return False
+ task = TaskManager().get_task(project.graph_build_task_id)
+ return bool(
+ task
+ and task.status in {TaskStatus.PENDING, TaskStatus.PROCESSING}
+ )
def allowed_file(filename: str) -> bool:
@@ -69,10 +163,38 @@ def list_projects():
@graph_bp.route('/project/<project_id>', methods=['DELETE'])
def delete_project(project_id: str):
+ with _project_build_lock(project_id):
+ return _delete_project_impl(project_id)
+
+
+def _delete_project_impl(project_id: str):
"""
删除项目
"""
- success = ProjectManager.delete_project(project_id)
+ project = ProjectManager.get_project(project_id)
+ if not project:
+ return jsonify({
+ "success": False,
+ "error": t('api.projectNotFound', id=project_id)
+ }), 404
+ if _project_has_active_build(project):
+ return jsonify({
+ "success": False,
+ "error": t('api.graphBuilding')
+ }), 409
+
+ graph_id = project.graph_id
+ graph_guard = (
+ graph_lifecycle_lock(graph_id) if graph_id else nullcontext()
+ )
+ with graph_guard:
+ try:
+ _delete_cloud_graph_if_present(graph_id)
+ except GraphInUseError as error:
+ return jsonify({"success": False, "error": str(error)}), 409
+ # The local reference remains protected until it is removed, so a new
+ # simulation cannot claim the just-deleted graph in between.
+ success = ProjectManager.delete_project(project_id)
if not success:
return jsonify({
@@ -88,6 +210,11 @@ def delete_project(project_id: str):
@graph_bp.route('/project/<project_id>/reset', methods=['POST'])
def reset_project(project_id: str):
+ with _project_build_lock(project_id):
+ return _reset_project_impl(project_id)
+
+
+def _reset_project_impl(project_id: str):
"""
重置项目状态(用于重新构建图谱)
"""
@@ -99,16 +226,30 @@ def reset_project(project_id: str):
"error": t('api.projectNotFound', id=project_id)
}), 404
- # 重置到本体已生成状态
- if project.ontology:
- project.status = ProjectStatus.ONTOLOGY_GENERATED
- else:
- project.status = ProjectStatus.CREATED
-
- project.graph_id = None
- project.graph_build_task_id = None
- project.error = None
- ProjectManager.save_project(project)
+ if _project_has_active_build(project):
+ return jsonify({
+ "success": False,
+ "error": t('api.graphBuilding')
+ }), 409
+
+ graph_id = project.graph_id
+ graph_guard = (
+ graph_lifecycle_lock(graph_id) if graph_id else nullcontext()
+ )
+ with graph_guard:
+ try:
+ _delete_cloud_graph_if_present(graph_id)
+ except GraphInUseError as error:
+ return jsonify({"success": False, "error": str(error)}), 409
+
+ # 重置到本体已生成状态
+ if project.ontology:
+ project.status = ProjectStatus.ONTOLOGY_GENERATED
+ else:
+ project.status = ProjectStatus.CREATED
+
+ _clear_project_graph_reference(project)
+ ProjectManager.save_project(project)
return jsonify({
"success": True,
@@ -259,6 +400,17 @@ def generate_ontology():
@graph_bp.route('/build', methods=['POST'])
def build_graph():
+ """Serialize build claims for the same project within this process."""
+
+ data = request.get_json(silent=True) or {}
+ project_id = data.get("project_id")
+ if not project_id:
+ return _build_graph_impl()
+ with _project_build_lock(project_id):
+ return _build_graph_impl()
+
+
+def _build_graph_impl():
"""
接口2:根据project_id构建图谱
@@ -315,31 +467,89 @@ def build_graph():
# 检查项目状态
force = data.get('force', False) # 强制重新构建
-
- if project.status == ProjectStatus.CREATED:
+ if not isinstance(force, bool):
return jsonify({
"success": False,
- "error": t('api.ontologyNotGenerated')
+ "error": "force must be a JSON boolean"
}), 400
- if project.status == ProjectStatus.GRAPH_BUILDING and not force:
+ if project.status == ProjectStatus.CREATED:
return jsonify({
"success": False,
- "error": t('api.graphBuilding'),
- "task_id": project.graph_build_task_id
+ "error": t('api.ontologyNotGenerated')
}), 400
- # 如果强制重建,重置状态
- if force and project.status in [ProjectStatus.GRAPH_BUILDING, ProjectStatus.FAILED, ProjectStatus.GRAPH_COMPLETED]:
- project.status = ProjectStatus.ONTOLOGY_GENERATED
- project.graph_id = None
- project.graph_build_task_id = None
- project.error = None
+ resume_existing_batch = False
+ if project.status == ProjectStatus.GRAPH_BUILDING:
+ if _project_has_active_build(project):
+ return jsonify({
+ "success": True,
+ "data": {
+ "project_id": project_id,
+ "task_id": project.graph_build_task_id,
+ "graph_id": project.graph_id,
+ "reused": True,
+ "message": t('api.graphBuilding')
+ }
+ })
+
+ if (
+ not force
+ and project.graph_id
+ and project.zep_batch_id
+ and project.zep_batch_operation_id
+ ):
+ builder = GraphBuilderService(api_key=Config.ZEP_API_KEY)
+ batch_summary = builder.get_batch_summary(project.zep_batch_id)
+ if getattr(batch_summary, "status", None) in {
+ "queued",
+ "processing",
+ "succeeded",
+ }:
+ resume_existing_batch = True
+
+ if not resume_existing_batch:
+ project.status = ProjectStatus.FAILED
+ project.error = (
+ "Graph build task is no longer present; the persisted Zep "
+ "batch cannot be resumed automatically"
+ )
+ ProjectManager.save_project(project)
+ if not force:
+ return jsonify({
+ "success": False,
+ "error": project.error,
+ "task_id": project.graph_build_task_id,
+ "recoverable": True,
+ }), 409
+
+ if project.status == ProjectStatus.GRAPH_COMPLETED and not force:
+ return jsonify({
+ "success": True,
+ "data": {
+ "project_id": project_id,
+ "task_id": project.graph_build_task_id,
+ "graph_id": project.graph_id,
+ "reused": True,
+ "message": t('progress.graphBuildComplete')
+ }
+ })
# 获取配置
graph_name = data.get('graph_name', project.name or 'MiroFish Graph')
chunk_size = data.get('chunk_size', project.chunk_size or Config.DEFAULT_CHUNK_SIZE)
chunk_overlap = data.get('chunk_overlap', project.chunk_overlap or Config.DEFAULT_CHUNK_OVERLAP)
+ if not isinstance(chunk_size, int) or chunk_size <= 0:
+ return jsonify({"success": False, "error": "chunk_size must be a positive integer"}), 400
+ if (
+ not isinstance(chunk_overlap, int)
+ or chunk_overlap < 0
+ or chunk_overlap >= chunk_size
+ ):
+ return jsonify({
+ "success": False,
+ "error": "chunk_overlap must satisfy 0 <= chunk_overlap < chunk_size"
+ }), 400
# 更新项目配置
project.chunk_size = chunk_size
@@ -360,6 +570,22 @@ def build_graph():
"success": False,
"error": t('api.ontologyNotFound')
}), 400
+
+ # Only mutate Cloud state after the complete rebuild request validates.
+ if project.status == ProjectStatus.FAILED or (
+ force and project.status == ProjectStatus.GRAPH_COMPLETED
+ ):
+ graph_id_to_delete = project.graph_id
+ graph_guard = (
+ graph_lifecycle_lock(graph_id_to_delete)
+ if graph_id_to_delete
+ else nullcontext()
+ )
+ with graph_guard:
+ _delete_cloud_graph_if_present(graph_id_to_delete)
+ project.status = ProjectStatus.ONTOLOGY_GENERATED
+ _clear_project_graph_reference(project)
+ ProjectManager.save_project(project)
# 创建异步任务
task_manager = TaskManager()
@@ -400,49 +626,79 @@ def build_graph():
chunk_size=chunk_size,
overlap=chunk_overlap
)
+ builder.validate_batch_chunks(chunks, batch_size=350)
total_chunks = len(chunks)
- # 创建图谱
- task_manager.update_task(
- task_id,
- message=t('progress.creatingZepGraph'),
- progress=10
- )
- graph_id = builder.create_graph(name=graph_name)
-
- # 更新项目的graph_id
- project.graph_id = graph_id
- ProjectManager.save_project(project)
-
- # 设置本体
- task_manager.update_task(
- task_id,
- message=t('progress.settingOntology'),
- progress=15
- )
- builder.set_ontology(graph_id, ontology)
-
- # 添加文本(progress_callback 签名是 (msg, progress_ratio))
- def add_progress_callback(msg, progress_ratio):
- progress = 15 + int(progress_ratio * 40) # 15% - 55%
+ if resume_existing_batch:
+ graph_id = project.graph_id
+ operation_id = builder.build_operation_id(graph_id, chunks)
+ if operation_id != project.zep_batch_operation_id:
+ raise RuntimeError(
+ "Persisted Zep batch does not match the current graph input"
+ )
+ submission = BatchSubmission(
+ batch_id=project.zep_batch_id,
+ operation_id=operation_id,
+ episode_uuids=[],
+ item_count=total_chunks,
+ )
task_manager.update_task(
task_id,
- message=msg,
- progress=progress
+ message=t('progress.waitingZepProcess'),
+ progress=55,
+ )
+ else:
+ # 创建图谱
+ task_manager.update_task(
+ task_id,
+ message=t('progress.creatingZepGraph'),
+ progress=10
+ )
+
+ def remember_graph(graph_id):
+ project.graph_id = graph_id
+ ProjectManager.save_project(project)
+
+ graph_id = builder.create_graph(
+ name=graph_name,
+ graph_id_callback=remember_graph,
+ )
+
+ # 设置本体
+ task_manager.update_task(
+ task_id,
+ message=t('progress.settingOntology'),
+ progress=15
+ )
+ builder.set_ontology(graph_id, ontology)
+
+ # 添加文本(progress_callback 签名是 (msg, progress_ratio))
+ def add_progress_callback(msg, progress_ratio):
+ progress = 15 + int(progress_ratio * 40) # 15% - 55%
+ task_manager.update_task(
+ task_id,
+ message=msg,
+ progress=progress
+ )
+
+ task_manager.update_task(
+ task_id,
+ message=t('progress.addingChunks', count=total_chunks),
+ progress=15
+ )
+
+ def remember_batch(batch_id, operation_id):
+ project.zep_batch_id = batch_id
+ project.zep_batch_operation_id = operation_id
+ ProjectManager.save_project(project)
+
+ submission = builder.add_text_batches(
+ graph_id,
+ chunks,
+ batch_size=350,
+ progress_callback=add_progress_callback,
+ batch_created_callback=remember_batch,
)
-
- task_manager.update_task(
- task_id,
- message=t('progress.addingChunks', count=total_chunks),
- progress=15
- )
-
- episode_uuids = builder.add_text_batches(
- graph_id,
- chunks,
- batch_size=3,
- progress_callback=add_progress_callback
- )
# 等待Zep处理完成(查询每个episode的processed状态)
task_manager.update_task(
@@ -459,7 +715,7 @@ def build_graph():
progress=progress
)
- builder._wait_for_episodes(episode_uuids, wait_progress_callback)
+ builder._wait_for_batch(submission, wait_progress_callback)
# 获取图谱数据
task_manager.update_task(
@@ -469,44 +725,48 @@ def build_graph():
)
graph_data = builder.get_graph_data(graph_id)
- # 更新项目状态
- project.status = ProjectStatus.GRAPH_COMPLETED
- ProjectManager.save_project(project)
-
node_count = graph_data.get("node_count", 0)
edge_count = graph_data.get("edge_count", 0)
build_logger.info(f"[{task_id}] 图谱构建完成: graph_id={graph_id}, 节点={node_count}, 边={edge_count}")
-
- # 完成
- task_manager.update_task(
- task_id,
- status=TaskStatus.COMPLETED,
- message=t('progress.graphBuildComplete'),
- progress=100,
- result={
- "project_id": project_id,
- "graph_id": graph_id,
- "node_count": node_count,
- "edge_count": edge_count,
- "chunk_count": total_chunks
- }
- )
+
+ # Publish local project/task terminal state under the same
+ # lifecycle lock used by reset/delete/build claims. This
+ # prevents a deletion from interleaving between the two saves.
+ with _project_build_lock(project_id):
+ project.status = ProjectStatus.GRAPH_COMPLETED
+ project.error = None
+ ProjectManager.save_project(project)
+ task_manager.update_task(
+ task_id,
+ status=TaskStatus.COMPLETED,
+ message=t('progress.graphBuildComplete'),
+ progress=100,
+ result={
+ "project_id": project_id,
+ "graph_id": graph_id,
+ "node_count": node_count,
+ "edge_count": edge_count,
+ "chunk_count": total_chunks,
+ "zep_batch_id": submission.batch_id,
+ }
+ )
except Exception as e:
# 更新项目状态为失败
build_logger.error(f"[{task_id}] 图谱构建失败: {str(e)}")
build_logger.debug(traceback.format_exc())
- project.status = ProjectStatus.FAILED
- project.error = str(e)
- ProjectManager.save_project(project)
-
- task_manager.update_task(
- task_id,
- status=TaskStatus.FAILED,
- message=t('progress.buildFailed', error=str(e)),
- error=traceback.format_exc()
- )
+ with _project_build_lock(project_id):
+ project.status = ProjectStatus.FAILED
+ project.error = str(e)
+ ProjectManager.save_project(project)
+
+ task_manager.update_task(
+ task_id,
+ status=TaskStatus.FAILED,
+ message=t('progress.buildFailed', error=str(e)),
+ error=traceback.format_exc()
+ )
# 启动后台线程
thread = threading.Thread(target=build_task, daemon=True)
@@ -517,10 +777,13 @@ def build_graph():
"data": {
"project_id": project_id,
"task_id": task_id,
+ "resumed": resume_existing_batch,
"message": t('api.graphBuildStarted', taskId=task_id)
}
})
+ except GraphInUseError as e:
+ return jsonify({"success": False, "error": str(e)}), 409
except Exception as e:
return jsonify({
"success": False,
@@ -606,14 +869,45 @@ def delete_graph(graph_id: str):
"error": t('api.zepApiKeyMissing')
}), 500
- builder = GraphBuilderService(api_key=Config.ZEP_API_KEY)
- builder.delete_graph(graph_id)
+ projects = ProjectManager.find_projects_by_graph_id(graph_id)
+ if not projects:
+ return jsonify({
+ "success": False,
+ "error": "No local project references this graph"
+ }), 404
+ project_ids = sorted({project.project_id for project in projects})
+ with ExitStack() as stack:
+ for project_id in project_ids:
+ stack.enter_context(_project_build_lock(project_id))
+ stack.enter_context(graph_lifecycle_lock(graph_id))
+
+ # Re-read under all owning project locks so a concurrent build
+ # claim cannot appear between validation and Cloud deletion.
+ projects = ProjectManager.find_projects_by_graph_id(graph_id)
+ if any(_project_has_active_build(project) for project in projects):
+ return jsonify({
+ "success": False,
+ "error": t('api.graphBuilding')
+ }), 409
+
+ _delete_cloud_graph_if_present(graph_id)
+
+ for project in projects:
+ _clear_project_graph_reference(project)
+ project.status = (
+ ProjectStatus.ONTOLOGY_GENERATED
+ if project.ontology
+ else ProjectStatus.CREATED
+ )
+ ProjectManager.save_project(project)
return jsonify({
"success": True,
"message": t('api.graphDeleted', id=graph_id)
})
+ except GraphInUseError as e:
+ return jsonify({"success": False, "error": str(e)}), 409
except Exception as e:
return jsonify({
"success": False,
diff --git a/backend/app/api/report.py b/backend/app/api/report.py
index d7f2a4d0..a1e7dbcf 100644
--- a/backend/app/api/report.py
+++ b/backend/app/api/report.py
@@ -12,10 +12,17 @@ from . import report_bp
from ..config import Config
from ..services.report_agent import ReportAgent, ReportManager, ReportStatus
from ..services.simulation_manager import SimulationManager
-from ..models.project import ProjectManager
+from ..services.simulation_runner import SimulationRunner, RunnerStatus
+from ..services.zep_graph_memory_updater import ZepGraphMemoryManager
+from ..models.project import ProjectManager, ProjectStatus
from ..models.task import TaskManager, TaskStatus
from ..utils.logger import get_logger
from ..utils.locale import t, get_locale, set_locale
+from ..utils.zep_lifecycle import (
+ graph_lifecycle_lock,
+ register_graph_reader,
+ unregister_graph_reader,
+)
logger = get_logger('mirofish.api.report')
@@ -58,6 +65,11 @@ def generate_report():
}), 400
force_regenerate = data.get('force_regenerate', False)
+ if not isinstance(force_regenerate, bool):
+ return jsonify({
+ "success": False,
+ "error": "force_regenerate must be a JSON boolean",
+ }), 400
# 获取模拟信息
manager = SimulationManager()
@@ -69,21 +81,41 @@ def generate_report():
"error": t('api.simulationNotFound', id=simulation_id)
}), 404
- # 检查是否已有报告
- if not force_regenerate:
- existing_report = ReportManager.get_report_by_simulation(simulation_id)
- if existing_report and existing_report.status == ReportStatus.COMPLETED:
- return jsonify({
- "success": True,
- "data": {
- "simulation_id": simulation_id,
- "report_id": existing_report.report_id,
- "status": "completed",
- "message": t('api.reportAlreadyExists'),
- "already_generated": True
- }
- })
-
+ run_state = SimulationRunner.get_run_state(simulation_id)
+ updater = ZepGraphMemoryManager.get_updater(simulation_id)
+ active_statuses = {
+ RunnerStatus.STARTING,
+ RunnerStatus.RUNNING,
+ RunnerStatus.PAUSED,
+ RunnerStatus.STOPPING,
+ }
+ if updater is not None or (
+ run_state is not None and run_state.runner_status in active_statuses
+ ):
+ return jsonify({
+ "success": False,
+ "error": (
+ "Simulation or Zep graph ingestion is still active; "
+ "wait for a terminal run status before generating a report"
+ ),
+ "ingestion_pending": updater is not None,
+ }), 409
+ successful_terminal_statuses = {
+ RunnerStatus.COMPLETED,
+ RunnerStatus.STOPPED,
+ }
+ if (
+ run_state is None
+ or run_state.runner_status not in successful_terminal_statuses
+ ):
+ return jsonify({
+ "success": False,
+ "error": (
+ "A successfully completed or stopped simulation is required "
+ "before generating a report"
+ ),
+ }), 409
+
# 获取项目信息
project = ProjectManager.get_project(state.project_id)
if not project:
@@ -92,12 +124,26 @@ def generate_report():
"error": t('api.projectNotFound', id=state.project_id)
}), 404
- graph_id = state.graph_id or project.graph_id
+ if project.status != ProjectStatus.GRAPH_COMPLETED:
+ return jsonify({
+ "success": False,
+ "error": "The project graph must be completely built before reporting",
+ }), 409
+
+ graph_id = project.graph_id
if not graph_id:
return jsonify({
"success": False,
"error": t('api.missingGraphIdEnsure')
}), 400
+ if state.graph_id and state.graph_id != graph_id:
+ return jsonify({
+ "success": False,
+ "error": (
+ "The simulation references an older graph; prepare it "
+ "again before generating a report"
+ ),
+ }), 409
simulation_requirement = project.simulation_requirement
if not simulation_requirement:
@@ -110,74 +156,147 @@ def generate_report():
import uuid
report_id = f"report_{uuid.uuid4().hex[:12]}"
- # 创建异步任务
- task_manager = TaskManager()
- task_id = task_manager.create_task(
- task_type="report_generate",
- metadata={
- "simulation_id": simulation_id,
- "graph_id": graph_id,
- "report_id": report_id
- }
- )
-
- # Capture locale before spawning background thread
- current_locale = get_locale()
-
- # 定义后台任务
- def run_generate():
- set_locale(current_locale)
- try:
- task_manager.update_task(
- task_id,
- status=TaskStatus.PROCESSING,
- progress=0,
- message=t('api.initReportAgent')
+ # Register the background report as a graph reader under the same lock
+ # used by graph deletion and updater startup. A lock itself cannot be
+ # acquired in this request thread and released by the worker, so the
+ # durable reader registration is the cross-thread lease.
+ with graph_lifecycle_lock(graph_id):
+ refreshed_state = manager.get_simulation(simulation_id)
+ refreshed_project = (
+ ProjectManager.get_project(refreshed_state.project_id)
+ if refreshed_state
+ else None
+ )
+ refreshed_run_state = SimulationRunner.get_run_state(simulation_id)
+ refreshed_updater = ZepGraphMemoryManager.get_updater(simulation_id)
+ if (
+ refreshed_state is None
+ or refreshed_project is None
+ or refreshed_project.graph_id != graph_id
+ or refreshed_project.status != ProjectStatus.GRAPH_COMPLETED
+ or (
+ refreshed_state.graph_id
+ and refreshed_state.graph_id != graph_id
)
-
- # 创建Report Agent
- agent = ReportAgent(
- graph_id=graph_id,
- simulation_id=simulation_id,
- simulation_requirement=simulation_requirement
+ ):
+ return jsonify({
+ "success": False,
+ "error": "The project graph changed while reporting was starting",
+ }), 409
+ if refreshed_updater is not None or (
+ refreshed_run_state is not None
+ and refreshed_run_state.runner_status in active_statuses
+ ):
+ return jsonify({
+ "success": False,
+ "error": (
+ "Simulation or Zep graph ingestion became active; "
+ "retry after it reaches a terminal state"
+ ),
+ "ingestion_pending": refreshed_updater is not None,
+ }), 409
+ if (
+ refreshed_run_state is None
+ or refreshed_run_state.runner_status
+ not in successful_terminal_statuses
+ ):
+ return jsonify({
+ "success": False,
+ "error": (
+ "A successfully completed or stopped simulation is "
+ "required before generating a report"
+ ),
+ }), 409
+
+ # Cached-report reuse is now part of the same atomic barrier, so a
+ # concurrent rerun cannot make the returned report stale between
+ # the status check and response.
+ if not force_regenerate:
+ existing_report = ReportManager.get_report_by_simulation(
+ simulation_id
)
-
- # 进度回调
- def progress_callback(stage, progress, message):
+ if (
+ existing_report
+ and existing_report.status == ReportStatus.COMPLETED
+ ):
+ return jsonify({
+ "success": True,
+ "data": {
+ "simulation_id": simulation_id,
+ "report_id": existing_report.report_id,
+ "status": "completed",
+ "message": t('api.reportAlreadyExists'),
+ "already_generated": True
+ }
+ })
+
+ task_manager = TaskManager()
+ task_id = task_manager.create_task(
+ task_type="report_generate",
+ metadata={
+ "simulation_id": simulation_id,
+ "graph_id": graph_id,
+ "report_id": report_id
+ }
+ )
+ current_locale = get_locale()
+ register_graph_reader(graph_id, report_id)
+
+ def run_generate():
+ set_locale(current_locale)
+ try:
task_manager.update_task(
task_id,
- progress=progress,
- message=f"[{stage}] {message}"
+ status=TaskStatus.PROCESSING,
+ progress=0,
+ message=t('api.initReportAgent')
)
-
- # 生成报告(传入预先生成的 report_id)
- report = agent.generate_report(
- progress_callback=progress_callback,
- report_id=report_id
- )
-
- # 保存报告
- ReportManager.save_report(report)
-
- if report.status == ReportStatus.COMPLETED:
- task_manager.complete_task(
- task_id,
- result={
- "report_id": report.report_id,
- "simulation_id": simulation_id,
- "status": "completed"
- }
+
+ agent = ReportAgent(
+ graph_id=graph_id,
+ simulation_id=simulation_id,
+ simulation_requirement=simulation_requirement
)
- else:
- task_manager.fail_task(task_id, report.error or t('api.reportGenerateFailed'))
-
- except Exception as e:
- logger.error(f"报告生成失败: {str(e)}")
- task_manager.fail_task(task_id, str(e))
-
- # 启动后台线程
- thread = threading.Thread(target=run_generate, daemon=True)
- thread.start()
+
+ def progress_callback(stage, progress, message):
+ task_manager.update_task(
+ task_id,
+ progress=progress,
+ message=f"[{stage}] {message}"
+ )
+
+ report = agent.generate_report(
+ progress_callback=progress_callback,
+ report_id=report_id
+ )
+ ReportManager.save_report(report)
+
+ if report.status == ReportStatus.COMPLETED:
+ task_manager.complete_task(
+ task_id,
+ result={
+ "report_id": report.report_id,
+ "simulation_id": simulation_id,
+ "status": "completed"
+ }
+ )
+ else:
+ task_manager.fail_task(
+ task_id,
+ report.error or t('api.reportGenerateFailed')
+ )
+ except Exception as e:
+ logger.error(f"报告生成失败: {str(e)}")
+ task_manager.fail_task(task_id, str(e))
+ finally:
+ unregister_graph_reader(graph_id, report_id)
+
+ try:
+ thread = threading.Thread(target=run_generate, daemon=True)
+ thread.start()
+ except Exception:
+ unregister_graph_reader(graph_id, report_id)
+ raise
return jsonify({
"success": True,
diff --git a/backend/app/api/simulation.py b/backend/app/api/simulation.py
index 54842436..aae76788 100644
--- a/backend/app/api/simulation.py
+++ b/backend/app/api/simulation.py
@@ -5,6 +5,7 @@ Step2: Zep实体读取与过滤、OASIS模拟准备与运行(全程自动化
import os
import traceback
+from contextlib import nullcontext
from flask import request, jsonify, send_file
from . import simulation_bp
@@ -12,9 +13,15 @@ from ..config import Config
from ..services.zep_entity_reader import ZepEntityReader
from ..services.oasis_profile_generator import OasisProfileGenerator
from ..services.simulation_manager import SimulationManager, SimulationStatus
-from ..services.simulation_runner import SimulationRunner, RunnerStatus
+from ..services.simulation_runner import (
+ SimulationRunner,
+ RunnerStatus,
+ SimulationStopPending,
+)
+from ..services.zep_graph_memory_updater import ZepGraphMemoryManager
from ..utils.logger import get_logger
from ..utils.locale import t, get_locale, set_locale
+from ..utils.zep_lifecycle import get_graph_readers, graph_lifecycle_lock
from ..models.project import ProjectManager
logger = get_logger('mirofish.api.simulation')
@@ -1546,6 +1553,16 @@ def start_simulation():
max_rounds = data.get('max_rounds') # 可选:最大模拟轮数
enable_graph_memory_update = data.get('enable_graph_memory_update', False) # 可选:是否启用图谱记忆更新
force = data.get('force', False) # 可选:强制重新开始
+ if not isinstance(enable_graph_memory_update, bool):
+ return jsonify({
+ "success": False,
+ "error": "enable_graph_memory_update must be a JSON boolean",
+ }), 400
+ if not isinstance(force, bool):
+ return jsonify({
+ "success": False,
+ "error": "force must be a JSON boolean",
+ }), 400
# 验证 max_rounds 参数
if max_rounds is not None:
@@ -1586,31 +1603,67 @@ def start_simulation():
is_prepared, prepare_info = _check_simulation_prepared(simulation_id)
if is_prepared:
- # 准备工作已完成,检查是否有正在运行的进程
- if state.status == SimulationStatus.RUNNING:
- # 检查模拟进程是否真的在运行
- run_state = SimulationRunner.get_run_state(simulation_id)
- if run_state and run_state.runner_status.value == "running":
- # 进程确实在运行
- if force:
- # 强制模式:停止运行中的模拟
- logger.info(f"强制模式:停止运行中的模拟 {simulation_id}")
- try:
- SimulationRunner.stop_simulation(simulation_id)
- except Exception as e:
- logger.warning(f"停止模拟时出现警告: {str(e)}")
- else:
- return jsonify({
- "success": False,
- "error": t('api.simRunningForceHint')
- }), 400
+ run_state = SimulationRunner.get_run_state(simulation_id)
+ updater = ZepGraphMemoryManager.get_updater(simulation_id)
+ needs_finalization = bool(
+ run_state
+ and run_state.runner_status in {
+ RunnerStatus.RUNNING,
+ RunnerStatus.PAUSED,
+ RunnerStatus.STOPPING,
+ RunnerStatus.FAILED,
+ }
+ and (
+ run_state.runner_status
+ in {
+ RunnerStatus.RUNNING,
+ RunnerStatus.PAUSED,
+ RunnerStatus.STOPPING,
+ }
+ or updater is not None
+ )
+ )
+ if needs_finalization:
+ if not force:
+ return jsonify({
+ "success": False,
+ "error": t('api.simRunningForceHint')
+ }), 400
+ logger.info(f"强制模式:先完成旧模拟终止 {simulation_id}")
+ try:
+ stopped = SimulationRunner.stop_simulation(simulation_id)
+ except SimulationStopPending as error:
+ return jsonify({
+ "success": False,
+ "pending": True,
+ "error": str(error),
+ }), 409
+ except Exception as error:
+ return jsonify({
+ "success": False,
+ "error": (
+ "Cannot restart until the previous simulation "
+ f"finalizes safely: {error}"
+ ),
+ }), 409
+ if stopped.runner_status != RunnerStatus.STOPPED:
+ return jsonify({
+ "success": False,
+ "error": "Previous simulation did not reach STOPPED",
+ }), 409
# 如果是强制模式,清理运行日志
if force:
logger.info(f"强制模式:清理模拟日志 {simulation_id}")
cleanup_result = SimulationRunner.cleanup_simulation_logs(simulation_id)
if not cleanup_result.get("success"):
- logger.warning(f"清理日志时出现警告: {cleanup_result.get('errors')}")
+ return jsonify({
+ "success": False,
+ "error": (
+ "Failed to clean previous simulation logs: "
+ f"{cleanup_result.get('errors')}"
+ ),
+ }), 500
force_restarted = True
# 进程不存在或已结束,重置状态为 ready
@@ -1627,34 +1680,82 @@ def start_simulation():
# 获取图谱ID(用于图谱记忆更新)
graph_id = None
if enable_graph_memory_update:
- # 从模拟状态或项目中获取 graph_id
- graph_id = state.graph_id
- if not graph_id:
- # 尝试从项目中获取
- project = ProjectManager.get_project(state.project_id)
- if project:
- graph_id = project.graph_id
-
+ # The project is authoritative. A graph ID copied into an older
+ # simulation can outlive a project reset/rebuild and must not be
+ # used to resurrect writes to a deleted graph.
+ project = ProjectManager.get_project(state.project_id)
+ graph_id = project.graph_id if project else None
if not graph_id:
return jsonify({
"success": False,
"error": t('api.graphIdRequiredForMemory')
}), 400
-
- logger.info(f"启用图谱记忆更新: simulation_id={simulation_id}, graph_id={graph_id}")
-
- # 启动模拟
- run_state = SimulationRunner.start_simulation(
- simulation_id=simulation_id,
- platform=platform,
- max_rounds=max_rounds,
- enable_graph_memory_update=enable_graph_memory_update,
- graph_id=graph_id
+
+ graph_guard = (
+ graph_lifecycle_lock(graph_id)
+ if enable_graph_memory_update
+ else nullcontext()
)
-
- # 更新模拟状态
- state.status = SimulationStatus.RUNNING
- manager._save_simulation_state(state)
+ with graph_guard:
+ if enable_graph_memory_update:
+ # Re-read both references under the same per-graph lock used
+ # by reset/delete. Keep the lock through updater creation in
+ # start_simulation so check -> claim is atomic.
+ refreshed_state = manager.get_simulation(simulation_id)
+ refreshed_project = (
+ ProjectManager.get_project(refreshed_state.project_id)
+ if refreshed_state
+ else None
+ )
+ current_graph_id = (
+ refreshed_project.graph_id if refreshed_project else None
+ )
+ if current_graph_id != graph_id:
+ return jsonify({
+ "success": False,
+ "error": (
+ "The project graph changed while the simulation "
+ "was starting; retry after refreshing the project"
+ ),
+ }), 409
+ if (
+ refreshed_state.graph_id
+ and refreshed_state.graph_id != current_graph_id
+ ):
+ return jsonify({
+ "success": False,
+ "error": (
+ "The simulation references an older graph; "
+ "prepare it again before enabling graph memory"
+ ),
+ }), 409
+ active_reports = get_graph_readers(graph_id)
+ if active_reports:
+ return jsonify({
+ "success": False,
+ "error": (
+ "A report is currently reading this graph; wait "
+ "for report generation to finish before enabling "
+ "graph memory updates"
+ ),
+ "active_reports": active_reports,
+ }), 409
+ state = refreshed_state
+ logger.info(
+ "启用图谱记忆更新: simulation_id=%s, graph_id=%s",
+ simulation_id,
+ graph_id,
+ )
+
+ # 启动模拟。启用图谱写入时仍持有 graph_guard,直到 updater
+ # claim 与进程资源全部发布完成。
+ run_state = SimulationRunner.start_simulation(
+ simulation_id=simulation_id,
+ platform=platform,
+ max_rounds=max_rounds,
+ enable_graph_memory_update=enable_graph_memory_update,
+ graph_id=graph_id
+ )
response_data = run_state.to_dict()
if max_rounds:
@@ -1720,14 +1821,22 @@ def stop_simulation():
manager = SimulationManager()
state = manager.get_simulation(simulation_id)
if state:
- state.status = SimulationStatus.PAUSED
+ state.status = SimulationStatus.STOPPED
+ state.error = None
manager._save_simulation_state(state)
return jsonify({
"success": True,
"data": run_state.to_dict()
})
-
+
+ except SimulationStopPending as e:
+ return jsonify({
+ "success": False,
+ "pending": True,
+ "error": str(e),
+ }), 202
+
except ValueError as e:
return jsonify({
"success": False,
@@ -1736,6 +1845,14 @@ def stop_simulation():
except Exception as e:
logger.error(f"停止模拟失败: {str(e)}")
+ simulation_id = (request.get_json(silent=True) or {}).get('simulation_id')
+ if simulation_id:
+ manager = SimulationManager()
+ state = manager.get_simulation(simulation_id)
+ if state:
+ state.status = SimulationStatus.FAILED
+ state.error = str(e)
+ manager._save_simulation_state(state)
return jsonify({
"success": False,
"error": str(e),
diff --git a/backend/app/config.py b/backend/app/config.py
index 280b089f..534e3baf 100644
--- a/backend/app/config.py
+++ b/backend/app/config.py
@@ -14,6 +14,29 @@ else:
load_dotenv(override=True)
+def _parse_number(name, raw_value, *, default, cast):
+ """Parse a numeric setting without making module import fail."""
+
+ try:
+ return cast(raw_value), None
+ except (TypeError, ValueError):
+ return default, f"{name} must be a number"
+
+
+_zep_request_timeout, _zep_request_timeout_error = _parse_number(
+ "ZEP_REQUEST_TIMEOUT_SECONDS",
+ os.environ.get("ZEP_REQUEST_TIMEOUT_SECONDS", "30"),
+ default=30.0,
+ cast=float,
+)
+_zep_ingestion_timeout, _zep_ingestion_timeout_error = _parse_number(
+ "ZEP_INGESTION_TIMEOUT_SECONDS",
+ os.environ.get("ZEP_INGESTION_TIMEOUT_SECONDS", "600"),
+ default=600,
+ cast=int,
+)
+
+
class Config:
"""Flask配置类"""
@@ -31,6 +54,13 @@ class Config:
# Zep配置
ZEP_API_KEY = os.environ.get('ZEP_API_KEY')
+ ZEP_REQUEST_TIMEOUT_SECONDS = _zep_request_timeout
+ ZEP_INGESTION_TIMEOUT_SECONDS = _zep_ingestion_timeout
+ _ZEP_CONFIG_PARSE_ERRORS = tuple(
+ error
+ for error in (_zep_request_timeout_error, _zep_ingestion_timeout_error)
+ if error
+ )
# 文件上传配置
MAX_CONTENT_LENGTH = 50 * 1024 * 1024 # 50MB
@@ -68,6 +98,13 @@ class Config:
errors.append("LLM_API_KEY 未配置")
if not cls.ZEP_API_KEY:
errors.append("ZEP_API_KEY 未配置")
+ if os.environ.get("ZEP_API_URL"):
+ errors.append("ZEP_API_URL 不受支持;MiroFish 仅连接 Zep Cloud")
+ errors.extend(cls._ZEP_CONFIG_PARSE_ERRORS)
+ if cls.ZEP_REQUEST_TIMEOUT_SECONDS <= 0:
+ errors.append("ZEP_REQUEST_TIMEOUT_SECONDS 必须大于 0")
+ if cls.ZEP_INGESTION_TIMEOUT_SECONDS <= 0:
+ errors.append("ZEP_INGESTION_TIMEOUT_SECONDS 必须大于 0")
if cls.DEBUG:
import warnings
warnings.warn("Flask DEBUG mode is enabled. Do not use in production.", RuntimeWarning)
diff --git a/backend/app/models/project.py b/backend/app/models/project.py
index 08978937..fa63ab94 100644
--- a/backend/app/models/project.py
+++ b/backend/app/models/project.py
@@ -43,6 +43,8 @@ class Project:
# 图谱信息(接口2完成后填充)
graph_id: Optional[str] = None
graph_build_task_id: Optional[str] = None
+ zep_batch_id: Optional[str] = None
+ zep_batch_operation_id: Optional[str] = None
# 配置
simulation_requirement: Optional[str] = None
@@ -66,6 +68,8 @@ class Project:
"analysis_summary": self.analysis_summary,
"graph_id": self.graph_id,
"graph_build_task_id": self.graph_build_task_id,
+ "zep_batch_id": self.zep_batch_id,
+ "zep_batch_operation_id": self.zep_batch_operation_id,
"simulation_requirement": self.simulation_requirement,
"chunk_size": self.chunk_size,
"chunk_overlap": self.chunk_overlap,
@@ -91,6 +95,8 @@ class Project:
analysis_summary=data.get('analysis_summary'),
graph_id=data.get('graph_id'),
graph_build_task_id=data.get('graph_build_task_id'),
+ zep_batch_id=data.get('zep_batch_id'),
+ zep_batch_operation_id=data.get('zep_batch_operation_id'),
simulation_requirement=data.get('simulation_requirement'),
chunk_size=data.get('chunk_size', 500),
chunk_overlap=data.get('chunk_overlap', 50),
@@ -195,7 +201,7 @@ class ProjectManager:
return Project.from_dict(data)
@classmethod
- def list_projects(cls, limit: int = 50) -> List[Project]:
+ def list_projects(cls, limit: Optional[int] = 50) -> List[Project]:
"""
列出所有项目
@@ -216,7 +222,17 @@ class ProjectManager:
# 按创建时间倒序排序
projects.sort(key=lambda p: p.created_at, reverse=True)
- return projects[:limit]
+ return projects if limit is None else projects[:limit]
+
+ @classmethod
+ def find_projects_by_graph_id(cls, graph_id: str) -> List[Project]:
+ """Return every persisted project that references a Cloud graph."""
+
+ return [
+ project
+ for project in cls.list_projects(limit=None)
+ if project.graph_id == graph_id
+ ]
@classmethod
def delete_project(cls, project_id: str) -> bool:
@@ -302,4 +318,3 @@ class ProjectManager:
for f in os.listdir(files_dir)
if os.path.isfile(os.path.join(files_dir, f))
]
-
diff --git a/backend/app/services/graph_builder.py b/backend/app/services/graph_builder.py
index 4a12c148..33706914 100644
--- a/backend/app/services/graph_builder.py
+++ b/backend/app/services/graph_builder.py
@@ -3,15 +3,14 @@
接口2:使用Zep API构建Standalone Graph
"""
-import os
+import hashlib
import uuid
import time
import threading
from typing import Dict, Any, List, Optional, Callable
from dataclasses import dataclass
-from zep_cloud.client import Zep
-from zep_cloud import EpisodeData, EntityEdgeSourceTarget
+from zep_cloud import BatchAddItem, EntityEdgeSourceTarget, NotFoundError
from ..config import Config
from ..models.task import TaskManager, TaskStatus
@@ -20,6 +19,12 @@ from ..utils.ontology import (
MAX_ONTOLOGY_TYPES,
RESERVED_ONTOLOGY_ATTRIBUTE_NAMES,
normalize_ontology_attributes,
+ normalize_ontology_source_targets,
+)
+from ..utils.zep import (
+ call_zep_read_with_retry,
+ get_zep_client,
+ is_retryable_zep_error,
)
from .text_processor import TextProcessor
from ..utils.locale import t, get_locale, set_locale
@@ -42,6 +47,16 @@ class GraphInfo:
}
+@dataclass(frozen=True)
+class BatchSubmission:
+ """Durable identity for one Zep Batch API ingestion operation."""
+
+ batch_id: str
+ operation_id: str
+ episode_uuids: List[str]
+ item_count: int
+
+
class GraphBuilderService:
"""
图谱构建服务
@@ -53,7 +68,7 @@ class GraphBuilderService:
if not self.api_key:
raise ValueError("ZEP_API_KEY 未配置")
- self.client = Zep(api_key=self.api_key)
+ self.client = get_zep_client(self.api_key)
self.task_manager = TaskManager()
def build_graph_async(
@@ -63,7 +78,7 @@ class GraphBuilderService:
graph_name: str = "MiroFish Graph",
chunk_size: int = 500,
chunk_overlap: int = 50,
- batch_size: int = 3
+ batch_size: int = 350
) -> str:
"""
异步构建图谱
@@ -123,6 +138,12 @@ class GraphBuilderService:
message=t('progress.startBuildingGraph')
)
+ # Validate the complete ingestion payload before the first Cloud
+ # mutation, including this legacy service entry point.
+ chunks = TextProcessor.split_text(text, chunk_size, chunk_overlap)
+ self.validate_batch_chunks(chunks, batch_size=batch_size)
+ total_chunks = len(chunks)
+
# 1. 创建图谱
graph_id = self.create_graph(graph_name)
self.task_manager.update_task(
@@ -139,9 +160,7 @@ class GraphBuilderService:
message=t('progress.ontologySet')
)
- # 3. 文本分块
- chunks = TextProcessor.split_text(text, chunk_size, chunk_overlap)
- total_chunks = len(chunks)
+ # 3. 文本分块已在 Cloud mutation 前完成并验证
self.task_manager.update_task(
task_id,
progress=20,
@@ -149,7 +168,7 @@ class GraphBuilderService:
)
# 4. 分批发送数据
- episode_uuids = self.add_text_batches(
+ submission = self.add_text_batches(
graph_id, chunks, batch_size,
lambda msg, prog: self.task_manager.update_task(
task_id,
@@ -165,8 +184,8 @@ class GraphBuilderService:
message=t('progress.waitingZepProcess')
)
- self._wait_for_episodes(
- episode_uuids,
+ self._wait_for_batch(
+ submission,
lambda msg, prog: self.task_manager.update_task(
task_id,
progress=60 + int(prog * 0.3), # 60-90%
@@ -195,17 +214,100 @@ class GraphBuilderService:
error_msg = f"{str(e)}\n{traceback.format_exc()}"
self.task_manager.fail_task(task_id, error_msg)
- def create_graph(self, name: str) -> str:
- """创建Zep图谱(公开方法)"""
- graph_id = f"mirofish_{uuid.uuid4().hex[:16]}"
-
- self.client.graph.create(
- graph_id=graph_id,
- name=name,
- description="MiroFish Social Simulation Graph"
- )
-
+ def create_graph(
+ self,
+ name: str,
+ *,
+ graph_id: str | None = None,
+ graph_id_callback: Optional[Callable[[str], None]] = None,
+ ) -> str:
+ """Create a graph with a caller-durable ID and reconcile lost replies."""
+
+ graph_id = graph_id or f"mirofish_{uuid.uuid4().hex[:16]}"
+ # Persist the client-generated ID before the non-idempotent POST so a
+ # later reset can clean up a graph whose successful response was lost.
+ if graph_id_callback:
+ graph_id_callback(graph_id)
+
+ try:
+ self.client.graph.create(
+ graph_id=graph_id,
+ name=name,
+ description="MiroFish Social Simulation Graph"
+ )
+ except Exception as error:
+ if not is_retryable_zep_error(error):
+ raise
+ reconciliation_error = None
+ for attempt in range(3):
+ try:
+ call_zep_read_with_retry(
+ lambda: self.client.graph.get(graph_id),
+ operation_name=f"reconcile graph create {graph_id}",
+ )
+ reconciliation_error = None
+ break
+ except NotFoundError as not_found:
+ reconciliation_error = not_found
+ if attempt < 2:
+ time.sleep(attempt + 1)
+ except Exception as read_error:
+ reconciliation_error = read_error
+ break
+ if reconciliation_error is not None:
+ raise error from reconciliation_error
+
return graph_id
+
+ @staticmethod
+ def build_operation_id(graph_id: str, chunks: List[str]) -> str:
+ payload_hash = hashlib.sha256("\0".join(chunks).encode("utf-8")).hexdigest()
+ return hashlib.sha256(
+ f"{graph_id}:{payload_hash}".encode("utf-8")
+ ).hexdigest()
+
+ def _find_batch_by_operation_id(
+ self,
+ graph_id: str,
+ operation_id: str,
+ *,
+ max_attempts: int = 3,
+ ) -> Any | None:
+ """Find one server-created batch after an ambiguous create reply."""
+
+ for attempt in range(1, max_attempts + 1):
+ matches: List[Any] = []
+ cursor: int | None = None
+ seen_cursors: set[int] = set()
+ while True:
+ page = call_zep_read_with_retry(
+ lambda: self.client.batch.list(limit=100, cursor=cursor),
+ operation_name=f"reconcile batch create {operation_id}",
+ )
+ for batch in getattr(page, "batches", None) or []:
+ metadata = getattr(batch, "metadata", None) or {}
+ if (
+ metadata.get("mirofish_operation_id") == operation_id
+ and metadata.get("graph_id") == graph_id
+ ):
+ matches.append(batch)
+ next_cursor = getattr(page, "next_cursor", None)
+ if next_cursor is None:
+ break
+ if next_cursor == cursor or next_cursor in seen_cursors:
+ raise RuntimeError("Zep batch list cursor did not advance")
+ seen_cursors.add(next_cursor)
+ cursor = next_cursor
+
+ if len(matches) > 1:
+ raise RuntimeError(
+ f"Multiple Zep batches match operation {operation_id}; refusing ambiguity"
+ )
+ if matches:
+ return matches[0]
+ if attempt < max_attempts:
+ time.sleep(attempt)
+ return None
def set_ontology(self, graph_id: str, ontology: Dict[str, Any]):
"""设置图谱本体(公开方法)"""
@@ -278,7 +380,9 @@ class GraphBuilderService:
# 构建source_targets
source_targets = []
- for st in edge_def.get("source_targets", []):
+ for st in normalize_ontology_source_targets(
+ edge_def.get("source_targets", [])
+ ):
source_targets.append(
EntityEdgeSourceTarget(
source=st.get("source", "Entity"),
@@ -293,8 +397,8 @@ class GraphBuilderService:
if entity_types or edge_definitions:
self.client.graph.set_ontology(
graph_ids=[graph_id],
- # zep-cloud 3.13.0 iterates entities.items(), so an edge-only
- # ontology must pass an empty dictionary rather than None.
+ # Zep iterates entities.items(), so edge-only ontologies must
+ # pass an empty dictionary rather than None.
entities=entity_types,
edges=edge_definitions if edge_definitions else None,
)
@@ -303,13 +407,53 @@ class GraphBuilderService:
self,
graph_id: str,
chunks: List[str],
- batch_size: int = 3,
- progress_callback: Optional[Callable] = None
- ) -> List[str]:
- """分批添加文本到图谱,返回所有 episode 的 uuid 列表"""
- episode_uuids = []
+ batch_size: int = 350,
+ progress_callback: Optional[Callable] = None,
+ batch_created_callback: Optional[Callable[[str | None, str], None]] = None,
+ ) -> BatchSubmission:
+ """Submit document chunks through Zep's current Batch API.
+
+ Mutating calls are deliberately not retried: create/add are not
+ documented as idempotent, and an ambiguous replay can duplicate graph
+ episodes. The returned batch identity allows callers to persist and
+ reconcile the operation instead.
+ """
+
+ if not graph_id:
+ raise ValueError("graph_id is required")
+ self.validate_batch_chunks(chunks, batch_size=batch_size)
+
total_chunks = len(chunks)
-
+ operation_id = self.build_operation_id(graph_id, chunks)
+ if batch_created_callback:
+ # Journal the deterministic operation before the server-generated
+ # batch ID POST. This leaves enough identity for later diagnosis
+ # even if both the response and immediate list reconciliation fail.
+ batch_created_callback(None, operation_id)
+
+ try:
+ batch = self.client.batch.create(
+ metadata={
+ "mirofish_operation_id": operation_id,
+ "graph_id": graph_id,
+ "chunk_count": total_chunks,
+ }
+ )
+ except Exception as error:
+ if not is_retryable_zep_error(error):
+ raise
+ batch = self._find_batch_by_operation_id(graph_id, operation_id)
+ if batch is None:
+ raise RuntimeError(
+ "Zep batch creation is unconfirmed and no matching operation was found"
+ ) from error
+ batch_id = getattr(batch, "batch_id", None)
+ if not batch_id:
+ raise RuntimeError("Zep Batch API returned no batch_id")
+ if batch_created_callback:
+ batch_created_callback(batch_id, operation_id)
+
+ episode_uuids: List[str] = []
for i in range(0, total_chunks, batch_size):
batch_chunks = chunks[i:i + batch_size]
batch_num = i // batch_size + 1
@@ -322,34 +466,254 @@ class GraphBuilderService:
progress
)
- # 构建episode数据
- episodes = [
- EpisodeData(data=chunk, type="text")
- for chunk in batch_chunks
+ items = [
+ BatchAddItem(
+ type="graph_episode",
+ graph_id=graph_id,
+ data=chunk,
+ data_type="text",
+ source_description="MiroFish source document chunk",
+ metadata={
+ "mirofish_operation_id": operation_id,
+ "chunk_index": i + offset,
+ "chunk_sha256": hashlib.sha256(
+ chunk.encode("utf-8")
+ ).hexdigest(),
+ },
+ )
+ for offset, chunk in enumerate(batch_chunks)
]
-
- # 发送到Zep
+
+ expected_item_count = i + len(items)
try:
- batch_result = self.client.graph.add_batch(
- graph_id=graph_id,
- episodes=episodes
+ item_details = self.client.batch.add(
+ batch_id=batch_id,
+ items=items,
)
-
- # 收集返回的 episode uuid
- if batch_result and isinstance(batch_result, list):
- for ep in batch_result:
- ep_uuid = getattr(ep, 'uuid_', None) or getattr(ep, 'uuid', None)
- if ep_uuid:
- episode_uuids.append(ep_uuid)
-
- # 避免请求过快
- time.sleep(1)
-
except Exception as e:
if progress_callback:
progress_callback(t('progress.batchFailed', batch=batch_num, error=str(e)), 0)
- raise
-
+ if is_retryable_zep_error(e):
+ recovered_items = self._reconcile_batch_item_count(
+ batch_id,
+ expected_item_count,
+ )
+ recovered_indexes = {
+ getattr(item, "sequence_index", None)
+ for item in recovered_items
+ }
+ if (
+ len(recovered_items) == expected_item_count
+ and recovered_indexes == set(range(expected_item_count))
+ ):
+ item_details = recovered_items[i:expected_item_count]
+ else:
+ raise RuntimeError(
+ f"Zep batch {batch_id} item submission is unconfirmed; "
+ "the draft was not processed or replayed"
+ ) from e
+ else:
+ raise RuntimeError(
+ f"Zep batch {batch_id} item submission failed"
+ ) from e
+
+ if len(item_details or []) != len(items):
+ recovered_items = self._reconcile_batch_item_count(
+ batch_id,
+ expected_item_count,
+ )
+ recovered_indexes = {
+ getattr(item, "sequence_index", None)
+ for item in recovered_items
+ }
+ if (
+ len(recovered_items) == expected_item_count
+ and recovered_indexes == set(range(expected_item_count))
+ ):
+ item_details = recovered_items[i:expected_item_count]
+ else:
+ raise RuntimeError(
+ f"Zep batch {batch_id} acknowledged {len(item_details or [])} "
+ f"of {len(items)} items"
+ )
+ for item in item_details:
+ episode_uuid = getattr(item, "episode_uuid", None)
+ if episode_uuid:
+ episode_uuids.append(episode_uuid)
+
+ try:
+ self.client.batch.process(batch_id=batch_id)
+ except Exception as error:
+ # A process response can be lost after the server accepted it.
+ # Reconcile with a safe GET instead of issuing a second POST.
+ summary = call_zep_read_with_retry(
+ lambda: self.client.batch.get(batch_id=batch_id),
+ operation_name=f"reconcile batch {batch_id}",
+ )
+ if getattr(summary, "status", None) in {None, "draft"}:
+ raise RuntimeError(
+ f"Zep batch {batch_id} processing is unconfirmed"
+ ) from error
+
+ return BatchSubmission(
+ batch_id=batch_id,
+ operation_id=operation_id,
+ episode_uuids=episode_uuids,
+ item_count=total_chunks,
+ )
+
+ @staticmethod
+ def validate_batch_chunks(chunks: List[str], *, batch_size: int = 350) -> None:
+ """Validate every Batch API limit before the first Cloud mutation."""
+
+ if not chunks:
+ raise ValueError("At least one text chunk is required")
+ if not 1 <= batch_size <= 350:
+ raise ValueError("batch_size must be between 1 and 350")
+ if len(chunks) > 50_000:
+ raise ValueError("A Zep batch cannot contain more than 50,000 items")
+ oversized = [index for index, chunk in enumerate(chunks) if len(chunk) > 10_000]
+ if oversized:
+ raise ValueError(
+ f"Zep batch item exceeds 10,000 characters at chunk {oversized[0]}"
+ )
+
+ def _list_batch_items(self, batch_id: str) -> List[Any]:
+ items: List[Any] = []
+ cursor: int | None = None
+ seen_cursors: set[int] = set()
+ while True:
+ page = call_zep_read_with_retry(
+ lambda: self.client.batch.list_items(
+ batch_id=batch_id,
+ limit=100,
+ cursor=cursor,
+ ),
+ operation_name=f"list batch items {batch_id}",
+ )
+ items.extend(getattr(page, "items", None) or [])
+ next_cursor = getattr(page, "next_cursor", None)
+ if next_cursor is None:
+ break
+ if next_cursor == cursor or next_cursor in seen_cursors:
+ raise RuntimeError(f"Zep batch {batch_id} item cursor did not advance")
+ seen_cursors.add(next_cursor)
+ cursor = next_cursor
+ return items
+
+ def _reconcile_batch_item_count(
+ self,
+ batch_id: str,
+ expected_item_count: int,
+ *,
+ max_attempts: int = 3,
+ ) -> List[Any]:
+ """Allow a short propagation window after an ambiguous add reply."""
+
+ items: List[Any] = []
+ for attempt in range(1, max_attempts + 1):
+ items = self._list_batch_items(batch_id)
+ if len(items) >= expected_item_count:
+ return items
+ if attempt < max_attempts:
+ time.sleep(attempt)
+ return items
+
+ def get_batch_summary(self, batch_id: str) -> Any:
+ """Read a persisted batch identity for restart reconciliation."""
+
+ return call_zep_read_with_retry(
+ lambda: self.client.batch.get(batch_id=batch_id),
+ operation_name=f"get batch {batch_id}",
+ )
+
+ def _wait_for_batch(
+ self,
+ submission: BatchSubmission,
+ progress_callback: Optional[Callable] = None,
+ timeout: int | None = None,
+ ) -> List[str]:
+ """Wait for a Batch API terminal state and validate every item."""
+
+ timeout = timeout or Config.ZEP_INGESTION_TIMEOUT_SECONDS
+ start_time = time.time()
+ terminal_states = {"succeeded", "partial", "failed", "invalid", "canceled"}
+
+ while True:
+ if time.time() - start_time > timeout:
+ raise TimeoutError(
+ f"Zep batch {submission.batch_id} did not finish within {timeout}s"
+ )
+
+ summary = call_zep_read_with_retry(
+ lambda: self.client.batch.get(batch_id=submission.batch_id),
+ operation_name=f"poll batch {submission.batch_id}",
+ )
+ status = getattr(summary, "status", None)
+ progress = getattr(summary, "progress", None)
+ percent = float(getattr(progress, "percent_complete", 0) or 0) / 100
+ if progress_callback:
+ completed = int(getattr(progress, "succeeded_items", 0) or 0)
+ progress_callback(
+ t(
+ 'progress.zepProcessing',
+ completed=completed,
+ total=submission.item_count,
+ pending=max(submission.item_count - completed, 0),
+ elapsed=int(time.time() - start_time),
+ ),
+ min(max(percent, 0.0), 1.0),
+ )
+
+ if status in terminal_states:
+ break
+ time.sleep(3)
+
+ items = self._list_batch_items(submission.batch_id)
+ if status != "succeeded":
+ failed_items = [
+ item for item in items
+ if getattr(item, "status", None) not in {"succeeded", "skipped"}
+ ]
+ first_error = getattr(failed_items[0], "error", None) if failed_items else None
+ raise RuntimeError(
+ f"Zep batch {submission.batch_id} ended as {status}; "
+ f"failed_items={len(failed_items)}; first_error={first_error}"
+ )
+ if len(items) != submission.item_count:
+ raise RuntimeError(
+ f"Zep batch {submission.batch_id} contains {len(items)} items, "
+ f"expected {submission.item_count}"
+ )
+
+ ordered_items = sorted(
+ items,
+ key=lambda item: getattr(item, "sequence_index", 0) or 0,
+ )
+ episode_uuids: List[str] = []
+ for item in ordered_items:
+ item_status = getattr(item, "status", None)
+ episode_uuid = getattr(item, "episode_uuid", None)
+ source_uuid = getattr(item, "source_uuid", None)
+ if item_status != "succeeded" or not episode_uuid:
+ raise RuntimeError(
+ f"Zep batch {submission.batch_id} returned an incomplete item"
+ )
+ if source_uuid and source_uuid != episode_uuid:
+ raise RuntimeError(
+ f"Zep batch {submission.batch_id} returned mismatched episode UUIDs"
+ )
+ episode_uuids.append(episode_uuid)
+
+ if progress_callback:
+ progress_callback(
+ t(
+ 'progress.processingComplete',
+ completed=len(episode_uuids),
+ total=submission.item_count,
+ ),
+ 1.0,
+ )
return episode_uuids
def _wait_for_episodes(
@@ -379,21 +743,22 @@ class GraphBuilderService:
t('progress.episodesTimeout', completed=completed_count, total=total_episodes),
completed_count / total_episodes
)
- break
+ raise TimeoutError(
+ f"Zep episode processing timed out with "
+ f"{len(pending_episodes)} episode(s) still pending"
+ )
# 检查每个 episode 的处理状态
for ep_uuid in list(pending_episodes):
- try:
- episode = self.client.graph.episode.get(uuid_=ep_uuid)
- is_processed = getattr(episode, 'processed', False)
-
- if is_processed:
- pending_episodes.remove(ep_uuid)
- completed_count += 1
-
- except Exception as e:
- # 忽略单个查询错误,继续
- pass
+ episode = call_zep_read_with_retry(
+ lambda: self.client.graph.episode.get(uuid_=ep_uuid),
+ operation_name=f"poll episode {ep_uuid}",
+ )
+ is_processed = getattr(episode, 'processed', False)
+
+ if is_processed:
+ pending_episodes.remove(ep_uuid)
+ completed_count += 1
elapsed = int(time.time() - start_time)
if progress_callback:
diff --git a/backend/app/services/oasis_profile_generator.py b/backend/app/services/oasis_profile_generator.py
index add3a24f..de6ce6e3 100644
--- a/backend/app/services/oasis_profile_generator.py
+++ b/backend/app/services/oasis_profile_generator.py
@@ -16,12 +16,16 @@ from dataclasses import dataclass, field
from datetime import datetime
from openai import OpenAI
-from zep_cloud.client import Zep
-
from ..config import Config
from ..utils.logger import get_logger
from ..utils.locale import get_language_instruction, get_locale, set_locale, t
from ..utils.openai_chat_compat import create_chat_completion, extract_chat_completion_text
+from ..utils.zep import (
+ call_zep_read_with_retry,
+ get_zep_client,
+ is_retryable_zep_error,
+ normalize_zep_search_query,
+)
from .zep_entity_reader import EntityNode, ZepEntityReader
logger = get_logger('mirofish.oasis_profile')
@@ -263,7 +267,7 @@ class OasisProfileGenerator:
if self.zep_api_key:
try:
- self.zep_client = Zep(api_key=self.zep_api_key)
+ self.zep_client = get_zep_client(self.zep_api_key)
except Exception as e:
logger.warning(f"Zep客户端初始化失败: {e}")
@@ -372,57 +376,35 @@ class OasisProfileGenerator:
logger.debug(f"跳过Zep检索:未设置graph_id")
return results
- comprehensive_query = t('progress.zepSearchQuery', name=entity_name)
+ comprehensive_query = normalize_zep_search_query(
+ t('progress.zepSearchQuery', name=entity_name)
+ )
def search_edges():
"""搜索边(事实/关系)- 带重试机制"""
- max_retries = 3
- last_exception = None
- delay = 2.0
-
- for attempt in range(max_retries):
- try:
- return self.zep_client.graph.search(
+ return call_zep_read_with_retry(
+ lambda: self.zep_client.graph.search(
query=comprehensive_query,
graph_id=self.graph_id,
limit=30,
scope="edges",
reranker="rrf"
- )
- except Exception as e:
- last_exception = e
- if attempt < max_retries - 1:
- logger.debug(f"Zep边搜索第 {attempt + 1} 次失败: {str(e)[:80]}, 重试中...")
- time.sleep(delay)
- delay *= 2
- else:
- logger.debug(f"Zep边搜索在 {max_retries} 次尝试后仍失败: {e}")
- return None
+ ),
+ operation_name=f"profile edge search ({entity.uuid})",
+ )
def search_nodes():
"""搜索节点(实体摘要)- 带重试机制"""
- max_retries = 3
- last_exception = None
- delay = 2.0
-
- for attempt in range(max_retries):
- try:
- return self.zep_client.graph.search(
+ return call_zep_read_with_retry(
+ lambda: self.zep_client.graph.search(
query=comprehensive_query,
graph_id=self.graph_id,
limit=20,
scope="nodes",
reranker="rrf"
- )
- except Exception as e:
- last_exception = e
- if attempt < max_retries - 1:
- logger.debug(f"Zep节点搜索第 {attempt + 1} 次失败: {str(e)[:80]}, 重试中...")
- time.sleep(delay)
- delay *= 2
- else:
- logger.debug(f"Zep节点搜索在 {max_retries} 次尝试后仍失败: {e}")
- return None
+ ),
+ operation_name=f"profile node search ({entity.uuid})",
+ )
try:
# 并行执行edges和nodes搜索
@@ -431,8 +413,11 @@ class OasisProfileGenerator:
node_future = executor.submit(search_nodes)
# 获取结果
- edge_result = edge_future.result(timeout=30)
- node_result = node_future.result(timeout=30)
+ # Each request already has the configured HTTP timeout and
+ # typed retry budget. A second hard-coded 30s future timeout
+ # discarded late successes while the executor still waited.
+ edge_result = edge_future.result()
+ node_result = node_future.result()
# 处理边搜索结果
all_facts = set()
@@ -462,10 +447,10 @@ class OasisProfileGenerator:
logger.info(f"Zep混合检索完成: {entity_name}, 获取 {len(results['facts'])} 条事实, {len(results['node_summaries'])} 个相关节点")
- except concurrent.futures.TimeoutError:
- logger.warning(f"Zep检索超时 ({entity_name})")
except Exception as e:
logger.warning(f"Zep检索失败 ({entity_name}): {e}")
+ if not is_retryable_zep_error(e):
+ raise
return results
diff --git a/backend/app/services/ontology_generator.py b/backend/app/services/ontology_generator.py
index b719a8dc..ce0ea9ab 100644
--- a/backend/app/services/ontology_generator.py
+++ b/backend/app/services/ontology_generator.py
@@ -13,6 +13,7 @@ from ..utils.file_parser import split_text_into_chunks
from ..utils.ontology import (
MAX_ONTOLOGY_TYPES,
normalize_ontology_attributes,
+ normalize_ontology_source_targets,
)
logger = logging.getLogger(__name__)
@@ -31,6 +32,18 @@ def _to_pascal_case(name: str) -> str:
return result if result else 'Unknown'
+def _to_upper_snake_case(name: str) -> str:
+ """Convert free-form or camelCase names to SCREAMING_SNAKE_CASE."""
+
+ separated = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', name.strip())
+ normalized = re.sub(r'[^a-zA-Z0-9]+', '_', separated).strip('_').upper()
+ if not normalized:
+ return "UNKNOWN"
+ if normalized[0].isdigit():
+ normalized = f"REL_{normalized}"
+ return normalized
+
+
# 本体生成的系统提示词
ONTOLOGY_SYSTEM_PROMPT = """你是一个专业的知识图谱本体设计专家。你的任务是分析给定的文本内容和模拟需求,设计适合**社交媒体舆论模拟**的实体类型和关系类型。
@@ -414,76 +427,71 @@ class OntologyGenerator:
def _validate_and_process(self, result: Dict[str, Any]) -> Dict[str, Any]:
"""验证和后处理结果"""
-
- # 确保必要字段存在
- if "entity_types" not in result:
- result["entity_types"] = []
- if "edge_types" not in result:
- result["edge_types"] = []
- if "analysis_summary" not in result:
+ if not isinstance(result, dict):
+ raise ValueError("Ontology result must be an object")
+
+ raw_entities = result.get("entity_types")
+ raw_edges = result.get("edge_types")
+ if not isinstance(raw_entities, list):
+ raw_entities = []
+ if not isinstance(raw_edges, list):
+ raw_edges = []
+ if not isinstance(result.get("analysis_summary"), str):
result["analysis_summary"] = ""
-
- # 验证实体类型
- # 记录原始名称到 PascalCase 的映射,用于后续修正 edge 的 source_targets 引用
- entity_name_map = {}
- for entity in result["entity_types"]:
- # 强制将 entity name 转为 PascalCase(Zep API 要求)
- if "name" in entity:
- original_name = entity["name"]
- entity["name"] = _to_pascal_case(original_name)
- if entity["name"] != original_name:
- logger.warning(f"Entity type name '{original_name}' auto-converted to '{entity['name']}'")
- entity_name_map[original_name] = entity["name"]
- # Normalize LLM output, enforce Zep field limits, and guarantee at
- # least one property for every custom ontology type.
+
+ # Normalize entity entries before touching their fields. LLMs
+ # occasionally emit a bare string, null, or another scalar.
+ entity_name_map: Dict[str, str] = {}
+ processed_entities: List[Dict[str, Any]] = []
+ seen_entity_names = set()
+ for raw_entity in raw_entities:
+ if isinstance(raw_entity, str):
+ entity = {"name": raw_entity}
+ elif isinstance(raw_entity, dict):
+ entity = dict(raw_entity)
+ else:
+ logger.warning("Ignoring non-object ontology entity entry")
+ continue
+
+ original_name = entity.get("name")
+ if not isinstance(original_name, str) or not original_name.strip():
+ logger.warning("Ignoring ontology entity without a usable name")
+ continue
+ original_name = original_name.strip()
+ normalized_name = _to_pascal_case(original_name)
+ if normalized_name == "Unknown":
+ continue
+ if normalized_name in seen_entity_names:
+ logger.warning(f"Duplicate entity type '{normalized_name}' removed during validation")
+ entity_name_map[original_name] = normalized_name
+ entity_name_map[original_name.lower()] = normalized_name
+ continue
+
+ if normalized_name != original_name:
+ logger.warning(
+ f"Entity type name '{original_name}' auto-converted to '{normalized_name}'"
+ )
+ entity["name"] = normalized_name
entity["attributes"] = normalize_ontology_attributes(
entity.get("attributes", [])
)
- if "examples" not in entity:
+ if not isinstance(entity.get("examples"), list):
entity["examples"] = []
- # 确保description不超过100字符
- if len(entity.get("description", "")) > 100:
- entity["description"] = entity["description"][:97] + "..."
-
- # 验证关系类型
- for edge in result["edge_types"]:
- # 强制将 edge name 转为 SCREAMING_SNAKE_CASE(Zep API 要求)
- if "name" in edge:
- original_name = edge["name"]
- edge["name"] = original_name.upper()
- if edge["name"] != original_name:
- logger.warning(f"Edge type name '{original_name}' auto-converted to '{edge['name']}'")
- # 修正 source_targets 中的实体名称引用,与转换后的 PascalCase 保持一致
- for st in edge.get("source_targets", []):
- if st.get("source") in entity_name_map:
- st["source"] = entity_name_map[st["source"]]
- if st.get("target") in entity_name_map:
- st["target"] = entity_name_map[st["target"]]
- if "source_targets" not in edge:
- edge["source_targets"] = []
- # Normalize LLM output, enforce Zep field limits, and guarantee at
- # least one property for every custom ontology type.
- edge["attributes"] = normalize_ontology_attributes(
- edge.get("attributes", [])
+ description = entity.get("description")
+ if not isinstance(description, str) or not description:
+ description = f"A {normalized_name} entity."
+ entity["description"] = (
+ description[:97] + "..." if len(description) > 100 else description
)
- if len(edge.get("description", "")) > 100:
- edge["description"] = edge["description"][:97] + "..."
-
- # Zep API 限制:最多 10 个自定义实体类型,最多 10 个自定义边类型
- MAX_ENTITY_TYPES = MAX_ONTOLOGY_TYPES
- MAX_EDGE_TYPES = MAX_ONTOLOGY_TYPES
-
- # 去重:按 name 去重,保留首次出现的
- seen_names = set()
- deduped = []
- for entity in result["entity_types"]:
- name = entity.get("name", "")
- if name and name not in seen_names:
- seen_names.add(name)
- deduped.append(entity)
- elif name in seen_names:
- logger.warning(f"Duplicate entity type '{name}' removed during validation")
- result["entity_types"] = deduped
+
+ seen_entity_names.add(normalized_name)
+ processed_entities.append(entity)
+ entity_name_map[original_name] = normalized_name
+ entity_name_map[original_name.lower()] = normalized_name
+ entity_name_map[normalized_name] = normalized_name
+ entity_name_map[normalized_name.lower()] = normalized_name
+
+ result["entity_types"] = processed_entities
# 兜底类型定义
person_fallback = {
@@ -523,9 +531,9 @@ class OntologyGenerator:
needed_slots = len(fallbacks_to_add)
# 如果添加后会超过 10 个,需要移除一些现有类型
- if current_count + needed_slots > MAX_ENTITY_TYPES:
+ if current_count + needed_slots > MAX_ONTOLOGY_TYPES:
# 计算需要移除多少个
- to_remove = current_count + needed_slots - MAX_ENTITY_TYPES
+ to_remove = current_count + needed_slots - MAX_ONTOLOGY_TYPES
# 从末尾移除(保留前面更重要的具体类型)
result["entity_types"] = result["entity_types"][:-to_remove]
@@ -533,11 +541,82 @@ class OntologyGenerator:
result["entity_types"].extend(fallbacks_to_add)
# 最终确保不超过限制(防御性编程)
- if len(result["entity_types"]) > MAX_ENTITY_TYPES:
- result["entity_types"] = result["entity_types"][:MAX_ENTITY_TYPES]
-
- if len(result["edge_types"]) > MAX_EDGE_TYPES:
- result["edge_types"] = result["edge_types"][:MAX_EDGE_TYPES]
+ result["entity_types"] = result["entity_types"][:MAX_ONTOLOGY_TYPES]
+
+ # Resolve edge endpoints only after entity fallback/capping, so an edge
+ # cannot refer to a type that was removed to satisfy Zep's limits.
+ valid_entity_names = {entity["name"] for entity in result["entity_types"]}
+ for name in valid_entity_names:
+ entity_name_map[name] = name
+ entity_name_map[name.lower()] = name
+
+ def resolve_entity_name(value: str) -> Optional[str]:
+ stripped = value.strip()
+ if stripped == "Entity":
+ return stripped
+ mapped = entity_name_map.get(stripped) or entity_name_map.get(stripped.lower())
+ if mapped in valid_entity_names:
+ return mapped
+ pascal_name = _to_pascal_case(stripped)
+ return pascal_name if pascal_name in valid_entity_names else None
+
+ processed_edges: List[Dict[str, Any]] = []
+ seen_edge_names = set()
+ for raw_edge in raw_edges:
+ if isinstance(raw_edge, str):
+ # A bare edge name has no endpoints and cannot be installed in
+ # Zep safely. Ignore it instead of inventing a relationship.
+ logger.warning(f"Ignoring ontology edge without source_targets: {raw_edge}")
+ continue
+ elif isinstance(raw_edge, dict):
+ edge = dict(raw_edge)
+ else:
+ logger.warning("Ignoring non-object ontology edge entry")
+ continue
+
+ original_name = edge.get("name")
+ if not isinstance(original_name, str) or not original_name.strip():
+ logger.warning("Ignoring ontology edge without a usable name")
+ continue
+ normalized_name = _to_upper_snake_case(original_name)
+ if normalized_name == "UNKNOWN" or normalized_name in seen_edge_names:
+ if normalized_name in seen_edge_names:
+ logger.warning(f"Duplicate edge type '{normalized_name}' removed during validation")
+ continue
+ if normalized_name != original_name:
+ logger.warning(
+ f"Edge type name '{original_name}' auto-converted to '{normalized_name}'"
+ )
+ edge["name"] = normalized_name
+
+ normalized_targets = []
+ for source_target in normalize_ontology_source_targets(
+ edge.get("source_targets", []),
+ limit=None,
+ ):
+ source = resolve_entity_name(source_target["source"])
+ target = resolve_entity_name(source_target["target"])
+ if source and target:
+ normalized_targets.append({"source": source, "target": target})
+ edge["source_targets"] = normalize_ontology_source_targets(
+ normalized_targets
+ )
+ edge["attributes"] = normalize_ontology_attributes(
+ edge.get("attributes", [])
+ )
+ description = edge.get("description")
+ if not isinstance(description, str) or not description:
+ description = f"A {normalized_name} relationship."
+ edge["description"] = (
+ description[:97] + "..." if len(description) > 100 else description
+ )
+
+ seen_edge_names.add(normalized_name)
+ processed_edges.append(edge)
+ if len(processed_edges) == MAX_ONTOLOGY_TYPES:
+ break
+
+ result["edge_types"] = processed_edges
return result
diff --git a/backend/app/services/simulation_manager.py b/backend/app/services/simulation_manager.py
index 76dc0c9a..73c32e0e 100644
--- a/backend/app/services/simulation_manager.py
+++ b/backend/app/services/simulation_manager.py
@@ -28,6 +28,7 @@ class SimulationStatus(str, Enum):
PREPARING = "preparing"
READY = "ready"
RUNNING = "running"
+ STOPPING = "stopping"
PAUSED = "paused"
STOPPED = "stopped" # 模拟被手动停止
COMPLETED = "completed" # 模拟自然完成
diff --git a/backend/app/services/simulation_runner.py b/backend/app/services/simulation_runner.py
index e86021f8..b3bdb39b 100644
--- a/backend/app/services/simulation_runner.py
+++ b/backend/app/services/simulation_runner.py
@@ -45,6 +45,10 @@ class RunnerStatus(str, Enum):
FAILED = "failed"
+class SimulationStopPending(TimeoutError):
+ """The monitor still owns a bounded graph-ingestion finalization."""
+
+
@dataclass
class AgentAction:
"""Agent动作记录"""
@@ -226,6 +230,56 @@ class SimulationRunner:
# 图谱记忆更新配置
_graph_memory_enabled: Dict[str, bool] = {} # simulation_id -> enabled
+ _finalization_locks: Dict[str, threading.Lock] = {}
+ _finalization_locks_guard = threading.Lock()
+ _manual_stop_requests: set[str] = set()
+
+ @classmethod
+ def _finalization_lock(cls, simulation_id: str) -> threading.Lock:
+ with cls._finalization_locks_guard:
+ return cls._finalization_locks.setdefault(
+ simulation_id, threading.Lock()
+ )
+
+ @classmethod
+ def _sync_simulation_status(
+ cls,
+ simulation_id: str,
+ runner_status: RunnerStatus,
+ error: str | None = None,
+ ) -> None:
+ """Keep persisted simulation metadata aligned with run_state.json."""
+
+ from .simulation_manager import SimulationManager, SimulationStatus
+
+ status_map = {
+ RunnerStatus.RUNNING: SimulationStatus.RUNNING,
+ RunnerStatus.STOPPING: SimulationStatus.STOPPING,
+ RunnerStatus.STOPPED: SimulationStatus.STOPPED,
+ RunnerStatus.COMPLETED: SimulationStatus.COMPLETED,
+ RunnerStatus.FAILED: SimulationStatus.FAILED,
+ }
+ status = status_map.get(runner_status)
+ if status is None:
+ return
+ try:
+ manager = SimulationManager()
+ simulation = manager.get_simulation(simulation_id)
+ if simulation is None:
+ return
+ simulation.status = status
+ simulation.error = error
+ manager._save_simulation_state(simulation)
+ except Exception as sync_error:
+ # state.json is a secondary projection. Never let a projection
+ # failure skip the authoritative run-state finalization or Zep
+ # ingestion drain.
+ logger.error(
+ "同步模拟状态失败: simulation_id=%s, status=%s, error=%s",
+ simulation_id,
+ runner_status.value,
+ sync_error,
+ )
@classmethod
def get_run_state(cls, simulation_id: str) -> Optional[SimulationRunState]:
@@ -331,11 +385,6 @@ class SimulationRunner:
Returns:
SimulationRunState
"""
- # 检查是否已在运行
- existing = cls.get_run_state(simulation_id)
- if existing and existing.runner_status in [RunnerStatus.RUNNING, RunnerStatus.STARTING]:
- raise ValueError(f"模拟已在运行中: {simulation_id}")
-
# 加载模拟配置
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
config_path = os.path.join(sim_dir, "simulation_config.json")
@@ -367,7 +416,22 @@ class SimulationRunner:
started_at=datetime.now().isoformat(),
)
- cls._save_run_state(state)
+ # Atomically claim this simulation ID. The expensive updater/process
+ # startup happens after releasing the lock, while the persisted
+ # STARTING state makes every concurrent start fail closed.
+ with cls._finalization_lock(simulation_id):
+ existing = cls.get_run_state(simulation_id)
+ active_statuses = {
+ RunnerStatus.STARTING,
+ RunnerStatus.RUNNING,
+ RunnerStatus.PAUSED,
+ RunnerStatus.STOPPING,
+ }
+ if (
+ existing and existing.runner_status in active_statuses
+ ) or ZepGraphMemoryManager.get_updater(simulation_id) is not None:
+ raise ValueError(f"模拟已在运行或结束处理中: {simulation_id}")
+ cls._save_run_state(state)
# 如果启用图谱记忆更新,创建更新器
if enable_graph_memory_update:
@@ -381,6 +445,16 @@ class SimulationRunner:
except Exception as e:
logger.error(f"创建图谱记忆更新器失败: {e}")
cls._graph_memory_enabled[simulation_id] = False
+ state.runner_status = RunnerStatus.FAILED
+ state.error = f"Zep图谱更新器初始化失败: {e}"
+ with cls._finalization_lock(simulation_id):
+ cls._save_run_state(state)
+ cls._sync_simulation_status(
+ simulation_id,
+ RunnerStatus.FAILED,
+ state.error,
+ )
+ raise RuntimeError(state.error) from e
else:
cls._graph_memory_enabled[simulation_id] = False
@@ -399,12 +473,35 @@ class SimulationRunner:
script_path = os.path.join(cls.SCRIPTS_DIR, script_name)
if not os.path.exists(script_path):
- raise ValueError(f"脚本不存在: {script_path}")
+ cleanup_error = None
+ if cls._graph_memory_enabled.get(simulation_id, False):
+ try:
+ ZepGraphMemoryManager.stop_updater(simulation_id)
+ cls._graph_memory_enabled.pop(simulation_id, None)
+ except Exception as error:
+ cleanup_error = error
+ state.runner_status = RunnerStatus.FAILED
+ state.twitter_running = False
+ state.reddit_running = False
+ state.error = f"脚本不存在: {script_path}"
+ if cleanup_error is not None:
+ state.error += f"; Zep图谱写入清理失败: {cleanup_error}"
+ with cls._finalization_lock(simulation_id):
+ cls._save_run_state(state)
+ cls._sync_simulation_status(
+ simulation_id,
+ RunnerStatus.FAILED,
+ state.error,
+ )
+ raise ValueError(state.error)
# 创建动作队列
action_queue = Queue()
cls._action_queues[simulation_id] = action_queue
-
+
+ process = None
+ main_log_file = None
+
# 启动模拟进程
try:
# 构建运行命令,使用完整路径
@@ -447,33 +544,70 @@ class SimulationRunner:
start_new_session=True, # 创建新进程组,确保服务器关闭时能终止所有相关进程
)
- # 保存文件句柄以便后续关闭
- cls._stdout_files[simulation_id] = main_log_file
- cls._stderr_files[simulation_id] = None # 不再需要单独的 stderr
-
- state.process_pid = process.pid
- state.runner_status = RunnerStatus.RUNNING
- cls._processes[simulation_id] = process
- cls._save_run_state(state)
-
# Capture locale before spawning monitor thread
current_locale = get_locale()
- # 启动监控线程
monitor_thread = threading.Thread(
target=cls._monitor_simulation,
args=(simulation_id, current_locale),
daemon=True
)
- monitor_thread.start()
- cls._monitor_threads[simulation_id] = monitor_thread
+
+ # Atomically publish every resource needed by stop/finalization.
+ # The monitor is registered before start; if it exits immediately,
+ # it waits on the same lock until RUNNING is fully visible.
+ with cls._finalization_lock(simulation_id):
+ cls._stdout_files[simulation_id] = main_log_file
+ cls._stderr_files[simulation_id] = None
+ state.process_pid = process.pid
+ state.runner_status = RunnerStatus.RUNNING
+ cls._processes[simulation_id] = process
+ cls._monitor_threads[simulation_id] = monitor_thread
+ cls._save_run_state(state)
+ cls._sync_simulation_status(
+ simulation_id,
+ RunnerStatus.RUNNING,
+ )
+ monitor_thread.start()
logger.info(f"模拟启动成功: {simulation_id}, pid={process.pid}, platform={platform}")
except Exception as e:
+ cleanup_errors = []
+ if process is not None and process.poll() is None:
+ try:
+ cls._terminate_process(process, simulation_id)
+ except Exception as error:
+ cleanup_errors.append(f"子进程终止失败: {error}")
+ cls._processes.pop(simulation_id, None)
+ cls._monitor_threads.pop(simulation_id, None)
+ cls._action_queues.pop(simulation_id, None)
+ cls._stdout_files.pop(simulation_id, None)
+ cls._stderr_files.pop(simulation_id, None)
+ if main_log_file is not None:
+ try:
+ main_log_file.close()
+ except Exception as error:
+ cleanup_errors.append(f"日志关闭失败: {error}")
+ if cls._graph_memory_enabled.get(simulation_id, False):
+ try:
+ ZepGraphMemoryManager.stop_updater(simulation_id)
+ cls._graph_memory_enabled.pop(simulation_id, None)
+ except Exception as error:
+ cleanup_errors.append(f"Zep图谱写入清理失败: {error}")
state.runner_status = RunnerStatus.FAILED
+ state.twitter_running = False
+ state.reddit_running = False
state.error = str(e)
- cls._save_run_state(state)
+ if cleanup_errors:
+ state.error += "; " + "; ".join(cleanup_errors)
+ with cls._finalization_lock(simulation_id):
+ cls._save_run_state(state)
+ cls._sync_simulation_status(
+ simulation_id,
+ RunnerStatus.FAILED,
+ state.error,
+ )
raise
return state
@@ -497,6 +631,8 @@ class SimulationRunner:
twitter_position = 0
reddit_position = 0
+ monitor_error: Exception | None = None
+ exit_code: int | None = None
try:
while process.poll() is None: # 进程仍在运行
# 读取 Twitter 动作日志
@@ -521,50 +657,93 @@ class SimulationRunner:
if os.path.exists(reddit_actions_log):
cls._read_action_log(reddit_actions_log, reddit_position, state, "reddit")
- # 进程结束
exit_code = process.returncode
- if exit_code == 0:
- state.runner_status = RunnerStatus.COMPLETED
- state.completed_at = datetime.now().isoformat()
- logger.info(f"模拟完成: {simulation_id}")
- else:
- state.runner_status = RunnerStatus.FAILED
- # 从主日志文件读取错误信息
- main_log_path = os.path.join(sim_dir, "simulation.log")
- error_info = ""
- try:
- if os.path.exists(main_log_path):
- with open(main_log_path, 'r', encoding='utf-8') as f:
- error_info = f.read()[-2000:] # 取最后2000字符
- except Exception:
- pass
- state.error = f"进程退出码: {exit_code}, 错误: {error_info}"
- logger.error(f"模拟失败: {simulation_id}, error={state.error}")
-
- state.twitter_running = False
- state.reddit_running = False
- cls._save_run_state(state)
-
except Exception as e:
logger.error(f"监控线程异常: {simulation_id}, error={str(e)}")
- state.runner_status = RunnerStatus.FAILED
- state.error = str(e)
- cls._save_run_state(state)
+ monitor_error = e
finally:
- # 停止图谱记忆更新器
- if cls._graph_memory_enabled.get(simulation_id, False):
- try:
- ZepGraphMemoryManager.stop_updater(simulation_id)
- logger.info(f"已停止图谱记忆更新: simulation_id={simulation_id}")
- except Exception as e:
- logger.error(f"停止图谱记忆更新器失败: {e}")
- cls._graph_memory_enabled.pop(simulation_id, None)
+ # Manual stop and natural completion can observe the same process
+ # exit. Serialize terminal state and updater drain so only one path
+ # owns the final result.
+ with cls._finalization_lock(simulation_id):
+ latest_state = cls.get_run_state(simulation_id)
+ if latest_state is not None:
+ state = latest_state
+
+ if state.runner_status not in {
+ RunnerStatus.STOPPED,
+ RunnerStatus.FAILED,
+ }:
+ manual_stop = simulation_id in cls._manual_stop_requests
+ desired_status = (
+ RunnerStatus.STOPPED
+ if manual_stop
+ else RunnerStatus.COMPLETED
+ )
+ error_message = None
+ if not manual_stop and monitor_error is not None:
+ desired_status = RunnerStatus.FAILED
+ error_message = str(monitor_error)
+ elif not manual_stop and exit_code != 0:
+ desired_status = RunnerStatus.FAILED
+ main_log_path = os.path.join(sim_dir, "simulation.log")
+ error_info = ""
+ try:
+ if os.path.exists(main_log_path):
+ with open(main_log_path, 'r', encoding='utf-8') as f:
+ error_info = f.read()[-2000:]
+ except Exception:
+ pass
+ error_message = (
+ f"进程退出码: {exit_code}, 错误: {error_info}"
+ )
+
+ state.twitter_running = False
+ state.reddit_running = False
+
+ if cls._graph_memory_enabled.get(simulation_id, False):
+ # STOPPING is a non-terminal ingestion barrier. The UI
+ # and report API must not observe COMPLETED until every
+ # accepted episode is processed by Zep Cloud.
+ state.runner_status = RunnerStatus.STOPPING
+ cls._save_run_state(state)
+ cls._sync_simulation_status(
+ simulation_id,
+ RunnerStatus.STOPPING,
+ )
+ try:
+ ZepGraphMemoryManager.stop_updater(simulation_id)
+ cls._graph_memory_enabled.pop(simulation_id, None)
+ logger.info(
+ "已停止图谱记忆更新: simulation_id=%s",
+ simulation_id,
+ )
+ except Exception as error:
+ logger.error(f"停止图谱记忆更新器失败: {error}")
+ desired_status = RunnerStatus.FAILED
+ error_message = f"Zep图谱写入未完整完成: {error}"
+
+ state.runner_status = desired_status
+ state.error = error_message
+ state.completed_at = datetime.now().isoformat()
+ cls._save_run_state(state)
+ cls._sync_simulation_status(
+ simulation_id,
+ desired_status,
+ error_message,
+ )
+ if desired_status == RunnerStatus.COMPLETED:
+ logger.info(f"模拟完成: {simulation_id}")
+ else:
+ logger.error(f"模拟失败: {simulation_id}, error={state.error}")
+ cls._manual_stop_requests.discard(simulation_id)
# 清理进程资源
cls._processes.pop(simulation_id, None)
cls._action_queues.pop(simulation_id, None)
+ cls._monitor_threads.pop(simulation_id, None)
# 关闭日志文件句柄
if simulation_id in cls._stdout_files:
@@ -635,9 +814,14 @@ class SimulationRunner:
# 如果运行了两个平台,需要两个都完成
all_completed = cls._check_all_platforms_completed(state)
if all_completed:
- state.runner_status = RunnerStatus.COMPLETED
- state.completed_at = datetime.now().isoformat()
- logger.info(f"所有平台模拟已完成: {state.simulation_id}")
+ # Platform completion is only an input
+ # signal. The monitor publishes the
+ # terminal status after the process has
+ # exited and Zep ingestion has drained.
+ logger.info(
+ f"所有平台已结束,等待进程与图谱写入完成: "
+ f"{state.simulation_id}"
+ )
# 更新轮次信息(从 round_end 事件)
elif event_type == "round_end":
@@ -776,51 +960,125 @@ class SimulationRunner:
@classmethod
def stop_simulation(cls, simulation_id: str) -> SimulationRunState:
"""停止模拟"""
- state = cls.get_run_state(simulation_id)
- if not state:
- raise ValueError(f"模拟不存在: {simulation_id}")
-
- if state.runner_status not in [RunnerStatus.RUNNING, RunnerStatus.PAUSED]:
- raise ValueError(f"模拟未在运行: {simulation_id}, status={state.runner_status}")
-
- state.runner_status = RunnerStatus.STOPPING
- cls._save_run_state(state)
-
- # 终止进程
- process = cls._processes.get(simulation_id)
- if process and process.poll() is None:
- try:
- cls._terminate_process(process, simulation_id)
- except ProcessLookupError:
- # 进程已经不存在
- pass
- except Exception as e:
- logger.error(f"终止进程组失败: {simulation_id}, error={e}")
- # 回退到直接终止进程
+ with cls._finalization_lock(simulation_id):
+ state = cls.get_run_state(simulation_id)
+ if not state:
+ raise ValueError(f"模拟不存在: {simulation_id}")
+ if state.runner_status == RunnerStatus.STOPPED:
+ return state
+
+ pending_updater = ZepGraphMemoryManager.get_updater(simulation_id)
+ retrying_finalization = (
+ pending_updater is not None
+ and state.runner_status in {
+ RunnerStatus.STOPPING,
+ RunnerStatus.FAILED,
+ }
+ )
+ if (
+ state.runner_status not in [
+ RunnerStatus.STARTING,
+ RunnerStatus.RUNNING,
+ RunnerStatus.PAUSED,
+ RunnerStatus.STOPPING,
+ ]
+ and not retrying_finalization
+ ):
+ raise ValueError(
+ f"模拟未在运行: {simulation_id}, status={state.runner_status}"
+ )
+
+ state.runner_status = RunnerStatus.STOPPING
+ cls._manual_stop_requests.add(simulation_id)
+ cls._save_run_state(state)
+ cls._sync_simulation_status(simulation_id, RunnerStatus.STOPPING)
+
+ # 终止进程
+ process = cls._processes.get(simulation_id)
+ if process and process.poll() is None:
try:
- process.terminate()
- process.wait(timeout=5)
- except Exception:
- process.kill()
-
- state.runner_status = RunnerStatus.STOPPED
- state.twitter_running = False
- state.reddit_running = False
- state.completed_at = datetime.now().isoformat()
- cls._save_run_state(state)
-
- # 停止图谱记忆更新器
- if cls._graph_memory_enabled.get(simulation_id, False):
- try:
- ZepGraphMemoryManager.stop_updater(simulation_id)
- logger.info(f"已停止图谱记忆更新: simulation_id={simulation_id}")
- except Exception as e:
- logger.error(f"停止图谱记忆更新器失败: {e}")
- cls._graph_memory_enabled.pop(simulation_id, None)
-
+ cls._terminate_process(process, simulation_id)
+ except ProcessLookupError:
+ pass
+ except Exception as e:
+ logger.error(f"终止进程组失败: {simulation_id}, error={e}")
+ try:
+ process.terminate()
+ process.wait(timeout=5)
+ except Exception:
+ process.kill()
+
+ # Let the monitor consume the final action-log tail and own the single
+ # updater drain. It will publish STOPPED (rather than COMPLETED) because
+ # the manual-stop marker is set above.
+ monitor = cls._monitor_threads.get(simulation_id)
+ if (
+ not retrying_finalization
+ and
+ monitor is not None
+ and monitor is not threading.current_thread()
+ and monitor.is_alive()
+ ):
+ wait_timeout = max(
+ 30.0,
+ Config.ZEP_INGESTION_TIMEOUT_SECONDS
+ + Config.ZEP_REQUEST_TIMEOUT_SECONDS
+ + 5,
+ )
+ monitor.join(timeout=wait_timeout)
+ if monitor.is_alive():
+ # The monitor still owns finalization and may be inside one
+ # bounded HTTP request. Do not block on or overwrite its lock;
+ # leave the observable state as STOPPING and let polling expose
+ # the eventual STOPPED/FAILED result.
+ raise SimulationStopPending(
+ f"模拟仍在停止中,图谱写入未在 {wait_timeout:.0f}s 内完成"
+ )
+ else:
+ # Restart recovery or tests may have no monitor thread. Complete
+ # the same barrier synchronously in this request.
+ with cls._finalization_lock(simulation_id):
+ state = cls.get_run_state(simulation_id) or state
+ if cls._graph_memory_enabled.get(simulation_id, False):
+ try:
+ ZepGraphMemoryManager.stop_updater(simulation_id)
+ cls._graph_memory_enabled.pop(simulation_id, None)
+ except Exception as error:
+ state.runner_status = RunnerStatus.FAILED
+ state.twitter_running = False
+ state.reddit_running = False
+ state.completed_at = datetime.now().isoformat()
+ state.error = f"Zep图谱写入未完整完成: {error}"
+ cls._save_run_state(state)
+ cls._sync_simulation_status(
+ simulation_id,
+ RunnerStatus.FAILED,
+ state.error,
+ )
+ raise RuntimeError(state.error) from error
+ state.runner_status = RunnerStatus.STOPPED
+ state.twitter_running = False
+ state.reddit_running = False
+ state.completed_at = datetime.now().isoformat()
+ state.error = None
+ cls._save_run_state(state)
+ cls._sync_simulation_status(
+ simulation_id,
+ RunnerStatus.STOPPED,
+ )
+ cls._manual_stop_requests.discard(simulation_id)
+
+ state = cls.get_run_state(simulation_id) or state
+ if state.runner_status == RunnerStatus.FAILED:
+ raise RuntimeError(state.error or "模拟停止失败")
+ if state.runner_status != RunnerStatus.STOPPED:
+ raise RuntimeError(
+ f"模拟停止未达到终态: {simulation_id}, status={state.runner_status}"
+ )
+
logger.info(f"模拟已停止: {simulation_id}")
return state
-
+
@classmethod
def _read_actions_from_file(
cls,
@@ -1194,95 +1452,99 @@ class SimulationRunner:
if cls._cleanup_done:
return
cls._cleanup_done = True
-
- # 检查是否有内容需要清理(避免空进程的进程打印无用日志)
- has_processes = bool(cls._processes)
- has_updaters = bool(cls._graph_memory_enabled)
-
- if not has_processes and not has_updaters:
- return # 没有需要清理的内容,静默返回
-
- logger.info("正在清理所有模拟进程...")
-
- # 首先停止所有图谱记忆更新器(stop_all 内部会打印日志)
- try:
- ZepGraphMemoryManager.stop_all()
- except Exception as e:
- logger.error(f"停止图谱记忆更新器失败: {e}")
- cls._graph_memory_enabled.clear()
-
- # 复制字典以避免在迭代时修改
- processes = list(cls._processes.items())
-
- for simulation_id, process in processes:
+
+ updater_ids = set(ZepGraphMemoryManager.get_simulation_ids())
+ simulation_ids = sorted(
+ set(cls._processes)
+ | set(cls._graph_memory_enabled)
+ | updater_ids
+ )
+ if not simulation_ids:
+ return
+
+ logger.info("正在安全完成所有模拟进程与图谱写入...")
+ cleanup_failed = False
+
+ # Each simulation follows the normal stop/finalization path: terminate
+ # its producer, let the monitor consume the final action-log tail, and
+ # only then drain Zep. This avoids dropping actions emitted during
+ # SIGTERM handling.
+ for simulation_id in simulation_ids:
try:
- if process.poll() is None: # 进程仍在运行
- logger.info(f"终止模拟进程: {simulation_id}, pid={process.pid}")
-
- try:
- # 使用跨平台的进程终止方法
+ state = cls.get_run_state(simulation_id)
+ updater = ZepGraphMemoryManager.get_updater(simulation_id)
+ process = cls._processes.get(simulation_id)
+
+ if state is None:
+ # Missing/corrupt state is exceptional, but retain the
+ # critical producer-before-consumer shutdown ordering.
+ if process is not None and process.poll() is None:
cls._terminate_process(process, simulation_id, timeout=5)
- except (ProcessLookupError, OSError):
- # 进程可能已经不存在,尝试直接终止
- try:
- process.terminate()
- process.wait(timeout=3)
- except Exception:
- process.kill()
-
- # 更新 run_state.json
- state = cls.get_run_state(simulation_id)
- if state:
- state.runner_status = RunnerStatus.STOPPED
- state.twitter_running = False
- state.reddit_running = False
- state.completed_at = datetime.now().isoformat()
- state.error = "服务器关闭,模拟被终止"
+ if updater is not None:
+ ZepGraphMemoryManager.stop_updater(simulation_id)
+ continue
+
+ if updater is not None:
+ cls._graph_memory_enabled[simulation_id] = True
+ if state.runner_status in {
+ RunnerStatus.IDLE,
+ RunnerStatus.STOPPED,
+ RunnerStatus.COMPLETED,
+ }:
+ # A retained updater means the old terminal projection
+ # was premature. Restore the ingestion barrier first.
+ state.runner_status = RunnerStatus.STOPPING
cls._save_run_state(state)
-
- # 同时更新 state.json,将状态设为 stopped
- try:
- sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
- state_file = os.path.join(sim_dir, "state.json")
- logger.info(f"尝试更新 state.json: {state_file}")
- if os.path.exists(state_file):
- with open(state_file, 'r', encoding='utf-8') as f:
- state_data = json.load(f)
- state_data['status'] = 'stopped'
- state_data['updated_at'] = datetime.now().isoformat()
- with open(state_file, 'w', encoding='utf-8') as f:
- json.dump(state_data, f, indent=2, ensure_ascii=False)
- logger.info(f"已更新 state.json 状态为 stopped: {simulation_id}")
- else:
- logger.warning(f"state.json 不存在: {state_file}")
- except Exception as state_err:
- logger.warning(f"更新 state.json 失败: {simulation_id}, error={state_err}")
-
- except Exception as e:
- logger.error(f"清理进程失败: {simulation_id}, error={e}")
-
- # 清理文件句柄
- for simulation_id, file_handle in list(cls._stdout_files.items()):
- try:
- if file_handle:
- file_handle.close()
- except Exception:
- pass
- cls._stdout_files.clear()
-
- for simulation_id, file_handle in list(cls._stderr_files.items()):
- try:
- if file_handle:
- file_handle.close()
- except Exception:
- pass
- cls._stderr_files.clear()
-
- # 清理内存中的状态
- cls._processes.clear()
- cls._action_queues.clear()
-
- logger.info("模拟进程清理完成")
+ cls._sync_simulation_status(
+ simulation_id,
+ RunnerStatus.STOPPING,
+ )
+
+ needs_finalization = bool(
+ (process is not None and process.poll() is None)
+ or updater is not None
+ or state.runner_status in {
+ RunnerStatus.STARTING,
+ RunnerStatus.RUNNING,
+ RunnerStatus.PAUSED,
+ RunnerStatus.STOPPING,
+ }
+ )
+ if needs_finalization:
+ cls.stop_simulation(simulation_id)
+
+ # A recovery path without a monitor does not run the monitor's
+ # resource cleanup block. Release only successfully stopped
+ # resources; FAILED/STOPPING resources remain retryable.
+ latest = cls.get_run_state(simulation_id)
+ if latest and latest.runner_status == RunnerStatus.STOPPED:
+ stopped_process = cls._processes.get(simulation_id)
+ if stopped_process is None or stopped_process.poll() is not None:
+ cls._processes.pop(simulation_id, None)
+ cls._action_queues.pop(simulation_id, None)
+ cls._monitor_threads.pop(simulation_id, None)
+ for file_map in (cls._stdout_files, cls._stderr_files):
+ file_handle = file_map.pop(simulation_id, None)
+ if file_handle:
+ try:
+ file_handle.close()
+ except Exception:
+ pass
+ except Exception as error:
+ cleanup_failed = True
+ logger.error(
+ "清理模拟失败,保留状态以便重试: simulation_id=%s, error=%s",
+ simulation_id,
+ error,
+ )
+
+ if cleanup_failed:
+ # Retained updaters and FAILED run states continue to block report
+ # generation and graph deletion. Permit an explicit retry.
+ cls._cleanup_done = False
+ logger.error("部分模拟未安全完成清理")
+ else:
+ logger.info("模拟进程与图谱写入清理完成")
@classmethod
def register_cleanup(cls):
@@ -1765,4 +2027,3 @@ class SimulationRunner:
results = results[:limit]
return results
-
diff --git a/backend/app/services/zep_entity_reader.py b/backend/app/services/zep_entity_reader.py
index d94c6c3f..1e20c6a3 100644
--- a/backend/app/services/zep_entity_reader.py
+++ b/backend/app/services/zep_entity_reader.py
@@ -3,15 +3,14 @@ Zep实体读取与过滤服务
从Zep图谱中读取节点,筛选出符合预定义实体类型的节点
"""
-import time
from typing import Dict, Any, List, Optional, Set, Callable, TypeVar
from dataclasses import dataclass, field
-
-from zep_cloud.client import Zep
+from zep_cloud import NotFoundError
from ..config import Config
from ..utils.logger import get_logger
from ..utils.zep_paging import fetch_all_nodes, fetch_all_edges
+from ..utils.zep import call_zep_read_with_retry, get_zep_client
logger = get_logger('mirofish.zep_entity_reader')
@@ -83,7 +82,7 @@ class ZepEntityReader:
if not self.api_key:
raise ValueError("ZEP_API_KEY 未配置")
- self.client = Zep(api_key=self.api_key)
+ self.client = get_zep_client(self.api_key)
def _call_with_retry(
self,
@@ -104,25 +103,12 @@ class ZepEntityReader:
Returns:
API调用结果
"""
- last_exception = None
- delay = initial_delay
-
- for attempt in range(max_retries):
- try:
- return func()
- except Exception as e:
- last_exception = e
- if attempt < max_retries - 1:
- logger.warning(
- f"Zep {operation_name} 第 {attempt + 1} 次尝试失败: {str(e)[:100]}, "
- f"{delay:.1f}秒后重试..."
- )
- time.sleep(delay)
- delay *= 2 # 指数退避
- else:
- logger.error(f"Zep {operation_name} 在 {max_retries} 次尝试后仍失败: {str(e)}")
-
- raise last_exception
+ return call_zep_read_with_retry(
+ func,
+ operation_name=operation_name,
+ max_attempts=max_retries,
+ initial_delay=initial_delay,
+ )
def get_all_nodes(self, graph_id: str) -> List[Dict[str, Any]]:
"""
@@ -209,8 +195,10 @@ class ZepEntityReader:
return edges_data
except Exception as e:
- logger.warning(f"获取节点 {node_uuid} 的边失败: {str(e)}")
- return []
+ # An empty edge list is valid data. Authentication, permission and
+ # transport failures must not be made indistinguishable from it.
+ logger.error(f"获取节点 {node_uuid} 的边失败: {str(e)}")
+ raise
def filter_defined_entities(
self,
@@ -406,9 +394,14 @@ class ZepEntityReader:
related_nodes=related_nodes,
)
+ except NotFoundError:
+ return None
except Exception as e:
+ # Only an actual Zep 404 means "entity not found". Propagate 401,
+ # 403 and exhausted transport errors so callers cannot prepare a
+ # simulation with silently incomplete graph context.
logger.error(f"获取实体 {entity_uuid} 失败: {str(e)}")
- return None
+ raise
def get_entities_by_type(
self,
@@ -433,4 +426,3 @@ class ZepEntityReader:
enrich_with_edges=enrich_with_edges
)
return result.entities
-
diff --git a/backend/app/services/zep_graph_memory_updater.py b/backend/app/services/zep_graph_memory_updater.py
index e034fee2..343ca1f2 100644
--- a/backend/app/services/zep_graph_memory_updater.py
+++ b/backend/app/services/zep_graph_memory_updater.py
@@ -3,20 +3,17 @@ Zep图谱记忆更新服务
将模拟中的Agent活动动态更新到Zep图谱中
"""
-import os
import time
import threading
-import json
-from typing import Dict, Any, List, Optional, Callable
+from typing import Dict, Any, List, Optional
from dataclasses import dataclass
from datetime import datetime
from queue import Queue, Empty
-from zep_cloud.client import Zep
-
from ..config import Config
from ..utils.logger import get_logger
from ..utils.locale import get_locale, set_locale
+from ..utils.zep import call_zep_read_with_retry, get_zep_client
logger = get_logger('mirofish.zep_graph_memory_updater')
@@ -58,8 +55,12 @@ class AgentActivity:
describe_func = action_descriptions.get(self.action_type, self._describe_generic)
description = describe_func()
- # 直接返回 "agent名称: 活动描述" 格式,不添加模拟前缀
- return f"{self.agent_name}: {description}"
+ # Keep the event time in the source text as well as episode metadata so
+ # temporal extraction does not collapse a multi-action batch.
+ return (
+ f"[{self.timestamp}] [{self.platform} round {self.round_num}] "
+ f"{self.agent_name}: {description}"
+ )
def _describe_create_post(self) -> str:
content = self.action_args.get("content", "")
@@ -199,6 +200,12 @@ class AgentActivity:
return f"执行了{self.action_type}操作"
+class _DrainDeadlineExceeded(TimeoutError):
+ def __init__(self, processed_count: int):
+ super().__init__("Zep updater drain deadline elapsed")
+ self.processed_count = processed_count
+
+
class ZepGraphMemoryUpdater:
"""
Zep图谱记忆更新器
@@ -225,11 +232,16 @@ class ZepGraphMemoryUpdater:
# 发送间隔(秒),避免请求过快
SEND_INTERVAL = 0.5
- # 重试配置
- MAX_RETRIES = 3
- RETRY_DELAY = 2 # 秒
+ # Zep recommends keeping an episode below 10,000 characters. Leave room
+ # for future source formatting changes.
+ MAX_EPISODE_CHARS = 9_500
- def __init__(self, graph_id: str, api_key: Optional[str] = None):
+ def __init__(
+ self,
+ graph_id: str,
+ api_key: Optional[str] = None,
+ simulation_id: Optional[str] = None,
+ ):
"""
初始化更新器
@@ -238,12 +250,13 @@ class ZepGraphMemoryUpdater:
api_key: Zep API Key(可选,默认从配置读取)
"""
self.graph_id = graph_id
+ self.simulation_id = simulation_id or "unknown"
self.api_key = api_key or Config.ZEP_API_KEY
if not self.api_key:
raise ValueError("ZEP_API_KEY未配置")
- self.client = Zep(api_key=self.api_key)
+ self.client = get_zep_client(self.api_key)
# 活动队列
self._activity_queue: Queue = Queue()
@@ -254,6 +267,7 @@ class ZepGraphMemoryUpdater:
'reddit': [],
}
self._buffer_lock = threading.Lock()
+ self._acceptance_lock = threading.Lock()
# 控制标志
self._running = False
@@ -265,6 +279,8 @@ class ZepGraphMemoryUpdater:
self._total_items_sent = 0 # 成功发送到Zep的活动条数
self._failed_count = 0 # 发送失败的批次数
self._skipped_count = 0 # 被过滤跳过的活动数(DO_NOTHING)
+ self._failed_batches: List[Dict[str, Any]] = []
+ self._pending_episode_uuids: List[str] = []
logger.info(f"ZepGraphMemoryUpdater 初始化完成: graph_id={graph_id}, batch_size={self.BATCH_SIZE}")
@@ -291,14 +307,34 @@ class ZepGraphMemoryUpdater:
logger.info(f"ZepGraphMemoryUpdater 已启动: graph_id={self.graph_id}")
def stop(self):
- """停止后台工作线程"""
- self._running = False
-
- # 发送剩余的活动
- self._flush_remaining()
-
+ """Drain the worker, flush tail events, and wait for Cloud ingestion."""
+ deadline = time.time() + Config.ZEP_INGESTION_TIMEOUT_SECONDS
+ # Serialize the accepting->closed transition with add_activity's
+ # check+enqueue operation. This closes the small race where a producer
+ # could enqueue after both the worker and final flush had exited.
+ with self._acceptance_lock:
+ self._running = False
+
if self._worker_thread and self._worker_thread.is_alive():
- self._worker_thread.join(timeout=10)
+ join_timeout = max(0.0, deadline - time.time())
+ self._worker_thread.join(timeout=join_timeout)
+ if self._worker_thread.is_alive():
+ raise TimeoutError(
+ f"Zep updater worker did not stop within {join_timeout:.0f}s"
+ )
+
+ # The worker has drained the queue. Only now is it safe to flush
+ # buffers; doing this before join loses an item already dequeued by the
+ # worker but not yet buffered.
+ self._flush_remaining(deadline=deadline)
+
+ if self._failed_batches:
+ raise RuntimeError(
+ f"{len(self._failed_batches)} Zep activity batch(es) failed; "
+ "simulation graph ingestion is incomplete"
+ )
+
+ self._wait_for_pending_episodes(deadline=deadline)
logger.info(f"ZepGraphMemoryUpdater 已停止: graph_id={self.graph_id}, "
f"total_activities={self._total_activities}, "
@@ -332,9 +368,12 @@ class ZepGraphMemoryUpdater:
if activity.action_type == "DO_NOTHING":
self._skipped_count += 1
return
-
- self._activity_queue.put(activity)
- self._total_activities += 1
+
+ with self._acceptance_lock:
+ if not self._running:
+ raise RuntimeError("Zep graph updater is not running")
+ self._activity_queue.put(activity)
+ self._total_activities += 1
logger.debug(f"添加活动到Zep队列: {activity.agent_name} - {activity.action_type}")
def add_activity_from_dict(self, data: Dict[str, Any], platform: str):
@@ -348,6 +387,9 @@ class ZepGraphMemoryUpdater:
# 跳过事件类型的条目
if "event_type" in data:
return
+ if data.get("success") is False:
+ self._skipped_count += 1
+ return
activity = AgentActivity(
platform=platform,
@@ -372,6 +414,7 @@ class ZepGraphMemoryUpdater:
# 将活动添加到对应平台的缓冲区
platform = activity.platform.lower()
+ batch = None
with self._buffer_lock:
if platform not in self._platform_buffers:
self._platform_buffers[platform] = []
@@ -381,10 +424,11 @@ class ZepGraphMemoryUpdater:
if len(self._platform_buffers[platform]) >= self.BATCH_SIZE:
batch = self._platform_buffers[platform][:self.BATCH_SIZE]
self._platform_buffers[platform] = self._platform_buffers[platform][self.BATCH_SIZE:]
- # 释放锁后再发送
- self._send_batch_activities(batch, platform)
- # 发送间隔,避免请求过快
- time.sleep(self.SEND_INTERVAL)
+
+ # Never hold the buffer lock across network I/O or sleep.
+ if batch:
+ self._send_batch_activities(batch, platform)
+ time.sleep(self.SEND_INTERVAL)
except Empty:
pass
@@ -393,7 +437,41 @@ class ZepGraphMemoryUpdater:
logger.error(f"工作循环异常: {e}")
time.sleep(1)
- def _send_batch_activities(self, activities: List[AgentActivity], platform: str):
+ def _build_episode_payloads(
+ self,
+ activities: List[AgentActivity],
+ ) -> List[tuple[List[AgentActivity], str]]:
+ payloads: List[tuple[List[AgentActivity], str]] = []
+ current_activities: List[AgentActivity] = []
+ current_lines: List[str] = []
+ current_length = 0
+
+ for activity in activities:
+ text = activity.to_episode_text()
+ if len(text) > self.MAX_EPISODE_CHARS:
+ marker = "... [truncated by MiroFish]"
+ text = text[: self.MAX_EPISODE_CHARS - len(marker)] + marker
+ projected_length = current_length + (1 if current_lines else 0) + len(text)
+ if current_lines and projected_length > self.MAX_EPISODE_CHARS:
+ payloads.append((current_activities, "\n".join(current_lines)))
+ current_activities = []
+ current_lines = []
+ current_length = 0
+ current_activities.append(activity)
+ current_lines.append(text)
+ current_length += (1 if len(current_lines) > 1 else 0) + len(text)
+
+ if current_lines:
+ payloads.append((current_activities, "\n".join(current_lines)))
+ return payloads
+
+ def _send_batch_activities(
+ self,
+ activities: List[AgentActivity],
+ platform: str,
+ *,
+ deadline: float | None = None,
+ ) -> int:
"""
批量发送活动到Zep图谱(合并为一条文本)
@@ -402,37 +480,80 @@ class ZepGraphMemoryUpdater:
platform: 平台名称
"""
if not activities:
- return
-
- # 将多条活动合并为一条文本,用换行分隔
- episode_texts = [activity.to_episode_text() for activity in activities]
- combined_text = "\n".join(episode_texts)
-
- # 带重试的发送
- for attempt in range(self.MAX_RETRIES):
+ return 0
+
+ processed_count = 0
+ for payload_activities, combined_text in self._build_episode_payloads(activities):
+ if deadline is not None and time.time() >= deadline:
+ raise _DrainDeadlineExceeded(processed_count)
try:
- self.client.graph.add(
+ episode = self.client.graph.add(
graph_id=self.graph_id,
type="text",
- data=combined_text
+ data=combined_text,
+ created_at=self._to_rfc3339(payload_activities[-1].timestamp),
+ source_description="MiroFish simulation activity batch",
+ metadata={
+ "source": "mirofish_simulation",
+ "simulation_id": self.simulation_id,
+ "platform": platform,
+ "activity_count": len(payload_activities),
+ "first_round": min(a.round_num for a in payload_activities),
+ "last_round": max(a.round_num for a in payload_activities),
+ "agent_ids": ",".join(
+ str(value)
+ for value in sorted({a.agent_id for a in payload_activities})
+ ),
+ "action_types": ",".join(
+ value
+ for value in sorted({a.action_type for a in payload_activities})
+ if value
+ ) or "unknown",
+ },
+ )
+
+ episode_uuid = (
+ getattr(episode, "uuid_", None)
+ or getattr(episode, "uuid", None)
)
-
+ if not episode_uuid:
+ raise RuntimeError("Zep graph.add returned no episode UUID")
+ self._pending_episode_uuids.append(str(episode_uuid))
self._total_sent += 1
- self._total_items_sent += len(activities)
+ self._total_items_sent += len(payload_activities)
display_name = self._get_platform_display_name(platform)
- logger.info(f"成功批量发送 {len(activities)} 条{display_name}活动到图谱 {self.graph_id}")
+ logger.info(f"成功批量发送 {len(payload_activities)} 条{display_name}活动到图谱 {self.graph_id}")
logger.debug(f"批量内容预览: {combined_text[:200]}...")
- return
-
+
except Exception as e:
- if attempt < self.MAX_RETRIES - 1:
- logger.warning(f"批量发送到Zep失败 (尝试 {attempt + 1}/{self.MAX_RETRIES}): {e}")
- time.sleep(self.RETRY_DELAY * (attempt + 1))
- else:
- logger.error(f"批量发送到Zep失败,已重试{self.MAX_RETRIES}次: {e}")
- self._failed_count += 1
-
- def _flush_remaining(self):
+ # graph.add has no idempotency key. Replaying an ambiguous
+ # response can duplicate extracted facts, so fail closed and
+ # surface the incomplete batch to SimulationRunner.
+ logger.error(f"批量发送到Zep失败,未自动重放非幂等写入: {e}")
+ self._failed_count += 1
+ self._failed_batches.append({
+ "platform": platform,
+ "activities": payload_activities,
+ "error": str(e),
+ })
+ finally:
+ # Successes have a confirmed episode UUID; failures are kept
+ # durably in _failed_batches and must never be replayed. Either
+ # way this payload is accounted for before moving on.
+ processed_count += len(payload_activities)
+ return processed_count
+
+ @staticmethod
+ def _to_rfc3339(value: str) -> str:
+ try:
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+ if parsed.tzinfo is None:
+ parsed = parsed.astimezone()
+ return parsed.isoformat()
+ except (AttributeError, TypeError, ValueError):
+ return datetime.now().astimezone().isoformat()
+
+ def _flush_remaining(self, *, deadline: float | None = None):
"""发送队列和缓冲区中剩余的活动"""
# 首先处理队列中剩余的活动,添加到缓冲区
while not self._activity_queue.empty():
@@ -446,16 +567,54 @@ class ZepGraphMemoryUpdater:
except Empty:
break
- # 然后发送各平台缓冲区中剩余的活动(即使不足BATCH_SIZE条)
- with self._buffer_lock:
- for platform, buffer in self._platform_buffers.items():
- if buffer:
- display_name = self._get_platform_display_name(platform)
- logger.info(f"发送{display_name}平台剩余的 {len(buffer)} 条活动")
- self._send_batch_activities(buffer, platform)
- # 清空所有缓冲区
- for platform in self._platform_buffers:
- self._platform_buffers[platform] = []
+ for platform in list(self._platform_buffers):
+ with self._buffer_lock:
+ buffer = list(self._platform_buffers.get(platform, []))
+ if not buffer:
+ continue
+ display_name = self._get_platform_display_name(platform)
+ logger.info(f"发送{display_name}平台剩余的 {len(buffer)} 条活动")
+ if deadline is not None and time.time() >= deadline:
+ raise TimeoutError(
+ "Zep updater drain deadline elapsed before flushing all activities"
+ )
+ try:
+ processed_count = self._send_batch_activities(
+ buffer,
+ platform,
+ deadline=deadline,
+ )
+ except _DrainDeadlineExceeded as error:
+ with self._buffer_lock:
+ del self._platform_buffers[platform][:error.processed_count]
+ raise TimeoutError(str(error)) from error
+ else:
+ with self._buffer_lock:
+ del self._platform_buffers[platform][:processed_count]
+
+ def _wait_for_pending_episodes(self, *, deadline: float | None = None) -> None:
+ pending = set(self._pending_episode_uuids)
+ if not pending:
+ return
+
+ if deadline is None:
+ deadline = time.time() + Config.ZEP_INGESTION_TIMEOUT_SECONDS
+ while pending:
+ if time.time() >= deadline:
+ raise TimeoutError(
+ f"Zep simulation ingestion timed out with {len(pending)} "
+ "episode(s) pending"
+ )
+ for episode_uuid in list(pending):
+ episode = call_zep_read_with_retry(
+ lambda: self.client.graph.episode.get(uuid_=episode_uuid),
+ operation_name=f"poll simulation episode {episode_uuid}",
+ )
+ if getattr(episode, "processed", False):
+ pending.remove(episode_uuid)
+ if pending:
+ time.sleep(3)
+ self._pending_episode_uuids = []
def get_stats(self) -> Dict[str, Any]:
"""获取统计信息"""
@@ -469,6 +628,7 @@ class ZepGraphMemoryUpdater:
"batches_sent": self._total_sent, # 成功发送的批次数
"items_sent": self._total_items_sent, # 成功发送的活动条数
"failed_count": self._failed_count, # 发送失败的批次数
+ "pending_episode_count": len(self._pending_episode_uuids),
"skipped_count": self._skipped_count, # 被过滤跳过的活动数(DO_NOTHING)
"queue_size": self._activity_queue.qsize(),
"buffer_sizes": buffer_sizes, # 各平台缓冲区大小
@@ -503,9 +663,13 @@ class ZepGraphMemoryManager:
if simulation_id in cls._updaters:
cls._updaters[simulation_id].stop()
- updater = ZepGraphMemoryUpdater(graph_id)
+ updater = ZepGraphMemoryUpdater(
+ graph_id,
+ simulation_id=simulation_id,
+ )
updater.start()
cls._updaters[simulation_id] = updater
+ cls._stop_all_done = False
logger.info(f"创建图谱记忆更新器: simulation_id={simulation_id}, graph_id={graph_id}")
return updater
@@ -513,16 +677,68 @@ class ZepGraphMemoryManager:
@classmethod
def get_updater(cls, simulation_id: str) -> Optional[ZepGraphMemoryUpdater]:
"""获取模拟的更新器"""
- return cls._updaters.get(simulation_id)
+ with cls._lock:
+ return cls._updaters.get(simulation_id)
+
+ @classmethod
+ def get_simulation_ids_for_graph(cls, graph_id: str) -> List[str]:
+ """Return simulations whose updater still owns or drains this graph."""
+
+ with cls._lock:
+ return sorted(
+ simulation_id
+ for simulation_id, updater in cls._updaters.items()
+ if updater.graph_id == graph_id
+ )
+
+ @classmethod
+ def get_simulation_ids(cls) -> List[str]:
+ """Return every simulation with a retained updater."""
+
+ with cls._lock:
+ return sorted(cls._updaters)
+
+ @classmethod
+ def discard_inactive_updater(cls, simulation_id: str) -> bool:
+ """Discard a failed, fully stopped updater during graph destruction."""
+
+ with cls._lock:
+ updater = cls._updaters.get(simulation_id)
+ if updater is None:
+ return False
+ worker_alive = bool(
+ updater._worker_thread and updater._worker_thread.is_alive()
+ )
+ if updater._running or worker_alive:
+ raise RuntimeError(
+ f"Zep updater for {simulation_id} is still active"
+ )
+ cls._updaters.pop(simulation_id, None)
+ logger.warning(
+ "Discarded incomplete Zep updater during explicit graph deletion: "
+ "simulation_id=%s, graph_id=%s",
+ simulation_id,
+ updater.graph_id,
+ )
+ return True
@classmethod
def stop_updater(cls, simulation_id: str):
"""停止并移除模拟的更新器"""
with cls._lock:
- if simulation_id in cls._updaters:
- cls._updaters[simulation_id].stop()
- del cls._updaters[simulation_id]
- logger.info(f"已停止图谱记忆更新器: simulation_id={simulation_id}")
+ updater = cls._updaters.get(simulation_id)
+ if updater is None:
+ return
+
+ # Do not hold the manager lock through up to several minutes of Cloud
+ # polling. Crucially, only remove the updater after a successful drain;
+ # on failure it remains visible to report/deletion barriers and can be
+ # stopped again.
+ updater.stop()
+ with cls._lock:
+ if cls._updaters.get(simulation_id) is updater:
+ cls._updaters.pop(simulation_id, None)
+ logger.info(f"已停止图谱记忆更新器: simulation_id={simulation_id}")
# 防止 stop_all 重复调用的标志
_stop_all_done = False
@@ -533,17 +749,34 @@ class ZepGraphMemoryManager:
# 防止重复调用
if cls._stop_all_done:
return
- cls._stop_all_done = True
-
+
+ with cls._lock:
+ simulation_ids = list(cls._updaters)
+
+ errors = []
+ for simulation_id in simulation_ids:
+ try:
+ cls.stop_updater(simulation_id)
+ except Exception as error:
+ # Keep a failed updater registered so the caller can retry and
+ # lifecycle/report guards still see the incomplete ingestion.
+ logger.error(
+ "停止更新器失败: simulation_id=%s, error=%s",
+ simulation_id,
+ error,
+ )
+ errors.append((simulation_id, error))
+
with cls._lock:
- if cls._updaters:
- for simulation_id, updater in list(cls._updaters.items()):
- try:
- updater.stop()
- except Exception as e:
- logger.error(f"停止更新器失败: simulation_id={simulation_id}, error={e}")
- cls._updaters.clear()
- logger.info("已停止所有图谱记忆更新器")
+ cls._stop_all_done = not cls._updaters
+
+ if errors:
+ details = "; ".join(
+ f"{simulation_id}: {error}"
+ for simulation_id, error in errors
+ )
+ raise RuntimeError(f"部分图谱更新器未完整停止: {details}")
+ logger.info("已停止所有图谱记忆更新器")
@classmethod
def get_all_stats(cls) -> Dict[str, Dict[str, Any]]:
diff --git a/backend/app/services/zep_tools.py b/backend/app/services/zep_tools.py
index 3bc8a57a..7535e394 100644
--- a/backend/app/services/zep_tools.py
+++ b/backend/app/services/zep_tools.py
@@ -12,14 +12,19 @@ import time
import json
from typing import Dict, Any, List, Optional
from dataclasses import dataclass, field
-
-from zep_cloud.client import Zep
+from zep_cloud import NotFoundError
from ..config import Config
from ..utils.logger import get_logger
from ..utils.llm_client import LLMClient
from ..utils.locale import get_locale, t
from ..utils.zep_paging import fetch_all_nodes, fetch_all_edges
+from ..utils.zep import (
+ call_zep_read_with_retry,
+ get_zep_client,
+ normalize_zep_search_limit,
+ normalize_zep_search_query,
+)
logger = get_logger('mirofish.zep_tools')
@@ -427,7 +432,7 @@ class ZepToolsService:
if not self.api_key:
raise ValueError("ZEP_API_KEY 未配置")
- self.client = Zep(api_key=self.api_key)
+ self.client = get_zep_client(self.api_key)
# LLM客户端用于InsightForge生成子问题
self._llm_client = llm_client
logger.info(t("console.zepToolsInitialized"))
@@ -440,26 +445,14 @@ class ZepToolsService:
return self._llm_client
def _call_with_retry(self, func, operation_name: str, max_retries: int = None):
- """带重试机制的API调用"""
- max_retries = max_retries or self.MAX_RETRIES
- last_exception = None
- delay = self.RETRY_DELAY
-
- for attempt in range(max_retries):
- try:
- return func()
- except Exception as e:
- last_exception = e
- if attempt < max_retries - 1:
- logger.warning(
- t("console.zepRetryAttempt", operation=operation_name, attempt=attempt + 1, error=str(e)[:100], delay=f"{delay:.1f}")
- )
- time.sleep(delay)
- delay *= 2
- else:
- logger.error(t("console.zepAllRetriesFailed", operation=operation_name, retries=max_retries, error=str(e)))
-
- raise last_exception
+ """Retry one safe read using typed Zep/HTTPX error classification."""
+
+ return call_zep_read_with_retry(
+ func,
+ operation_name=operation_name,
+ max_attempts=max_retries or self.MAX_RETRIES,
+ initial_delay=self.RETRY_DELAY,
+ )
def search_graph(
self,
@@ -485,13 +478,15 @@ class ZepToolsService:
"""
logger.info(t("console.graphSearch", graphId=graph_id, query=query[:50]))
- # 尝试使用Zep Cloud Search API
+ zep_query = normalize_zep_search_query(query)
+ zep_limit = normalize_zep_search_limit(limit)
+
try:
search_results = self._call_with_retry(
func=lambda: self.client.graph.search(
graph_id=graph_id,
- query=query,
- limit=limit,
+ query=zep_query,
+ limit=zep_limit,
scope=scope,
reranker="cross_encoder"
),
@@ -539,9 +534,10 @@ class ZepToolsService:
)
except Exception as e:
- logger.warning(t("console.zepSearchApiFallback", error=str(e)))
- # 降级:使用本地关键词匹配搜索
- return self._local_search(graph_id, query, limit, scope)
+ # Authentication, invalid input, missing graphs, and exhausted
+ # transient failures must remain visible to the report workflow.
+ logger.error(t("console.zepSearchApiFallback", error=str(e)))
+ raise
def _local_search(
self,
@@ -741,9 +737,11 @@ class ZepToolsService:
summary=node.summary or "",
attributes=node.attributes or {}
)
+ except NotFoundError:
+ return None
except Exception as e:
logger.error(t("console.fetchNodeDetailFailed", error=str(e)))
- return None
+ raise
def get_node_edges(self, graph_id: str, node_uuid: str) -> List[EdgeInfo]:
"""
@@ -774,8 +772,8 @@ class ZepToolsService:
return result
except Exception as e:
- logger.warning(t("console.fetchNodeEdgesFailed", error=str(e)))
- return []
+ logger.error(t("console.fetchNodeEdgesFailed", error=str(e)))
+ raise
def get_entities_by_type(
self,
diff --git a/backend/app/utils/ontology.py b/backend/app/utils/ontology.py
index d1596c68..632c9995 100644
--- a/backend/app/utils/ontology.py
+++ b/backend/app/utils/ontology.py
@@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional
MAX_ONTOLOGY_TYPES = 10
MAX_ONTOLOGY_ATTRIBUTES = 10
+MAX_ONTOLOGY_SOURCE_TARGETS = 10
RESERVED_ONTOLOGY_ATTRIBUTE_NAMES = frozenset({
"uuid",
"name",
@@ -67,3 +68,36 @@ def normalize_ontology_attributes(attributes: Any) -> List[Dict[str, Any]]:
normalized_attributes.append(dict(_FALLBACK_ATTRIBUTE))
return normalized_attributes
+
+
+def normalize_ontology_source_targets(
+ source_targets: Any,
+ *,
+ limit: int | None = MAX_ONTOLOGY_SOURCE_TARGETS,
+) -> List[Dict[str, str]]:
+ """Return unique, structurally valid source-target pairs within Zep limits."""
+
+ if not isinstance(source_targets, list):
+ return []
+
+ normalized_targets: List[Dict[str, str]] = []
+ seen: set[tuple[str, str]] = set()
+ for source_target in source_targets:
+ if not isinstance(source_target, dict):
+ continue
+ source = source_target.get("source")
+ target = source_target.get("target")
+ if not isinstance(source, str) or not source.strip():
+ continue
+ if not isinstance(target, str) or not target.strip():
+ continue
+
+ pair = (source.strip(), target.strip())
+ if pair in seen:
+ continue
+ seen.add(pair)
+ normalized_targets.append({"source": pair[0], "target": pair[1]})
+ if limit is not None and len(normalized_targets) == limit:
+ break
+
+ return normalized_targets
diff --git a/backend/app/utils/zep.py b/backend/app/utils/zep.py
new file mode 100644
index 00000000..16dbe20e
--- /dev/null
+++ b/backend/app/utils/zep.py
@@ -0,0 +1,158 @@
+"""Shared Zep Cloud client, request limits, and retry policy."""
+
+from __future__ import annotations
+
+import os
+import time
+from functools import lru_cache
+from typing import Any, Callable, TypeVar
+
+import httpx
+from zep_cloud.client import Zep
+from zep_cloud.core.api_error import ApiError as ZepApiError
+
+from ..config import Config
+from .logger import get_logger
+
+logger = get_logger("mirofish.zep")
+
+T = TypeVar("T")
+
+ZEP_CLOUD_BASE_URL = "https://api.getzep.com/api/v2"
+MAX_ZEP_SEARCH_QUERY_CHARS = 400
+MAX_ZEP_SEARCH_RESULTS = 50
+
+
+def normalize_zep_search_query(query: Any) -> str:
+ """Return a non-empty query within Zep Cloud's endpoint limit."""
+
+ if not isinstance(query, str):
+ raise ValueError("Zep search query must be a string")
+ normalized = query.strip()
+ if not normalized:
+ raise ValueError("Zep search query must not be empty")
+ return normalized[:MAX_ZEP_SEARCH_QUERY_CHARS]
+
+
+def normalize_zep_search_limit(limit: Any) -> int:
+ """Clamp a search result limit to the current Zep Cloud contract."""
+
+ try:
+ normalized = int(limit)
+ except (TypeError, ValueError) as exc:
+ raise ValueError("Zep search limit must be an integer") from exc
+ if normalized < 1:
+ raise ValueError("Zep search limit must be at least 1")
+ return min(normalized, MAX_ZEP_SEARCH_RESULTS)
+
+
+@lru_cache(maxsize=4)
+def _cached_zep_client(api_key: str, timeout: float) -> Zep:
+ return Zep(
+ api_key=api_key,
+ base_url=ZEP_CLOUD_BASE_URL,
+ timeout=timeout,
+ )
+
+
+def get_zep_client(api_key: str | None = None, timeout: float | None = None) -> Zep:
+ """Return a process-shared, explicitly configured Zep Cloud client."""
+
+ if Config._ZEP_CONFIG_PARSE_ERRORS:
+ raise ValueError("; ".join(Config._ZEP_CONFIG_PARSE_ERRORS))
+
+ # zep-cloud gives ZEP_API_URL precedence even when base_url is explicit.
+ # Reject it so this Cloud-only integration cannot silently target a
+ # self-hosted or compatibility endpoint.
+ if os.environ.get("ZEP_API_URL"):
+ raise ValueError("ZEP_API_URL is unsupported; unset it to use Zep Cloud")
+
+ normalized_key = (api_key or Config.ZEP_API_KEY or "").strip()
+ if not normalized_key:
+ raise ValueError("ZEP_API_KEY 未配置")
+
+ request_timeout = float(
+ timeout if timeout is not None else Config.ZEP_REQUEST_TIMEOUT_SECONDS
+ )
+ if request_timeout <= 0:
+ raise ValueError("Zep request timeout must be greater than 0")
+ return _cached_zep_client(normalized_key, request_timeout)
+
+
+def clear_zep_client_cache() -> None:
+ """Clear cached clients. Intended for tests and controlled reconfiguration."""
+
+ _cached_zep_client.cache_clear()
+
+
+def is_retryable_zep_error(error: BaseException) -> bool:
+ """Return whether a failed *read* is safe and useful to retry."""
+
+ if isinstance(error, (httpx.TimeoutException, httpx.TransportError)):
+ return True
+ if isinstance(error, (ConnectionError, TimeoutError, OSError)):
+ return True
+ if isinstance(error, ZepApiError):
+ status_code = error.status_code
+ return status_code in {408, 429} or (
+ status_code is not None and 500 <= status_code <= 599
+ )
+ return False
+
+
+def _retry_after_seconds(error: BaseException) -> float | None:
+ if not isinstance(error, ZepApiError) or not error.headers:
+ return None
+ value = next(
+ (
+ header_value
+ for header_name, header_value in error.headers.items()
+ if header_name.lower() == "retry-after"
+ ),
+ None,
+ )
+ if value is None:
+ return None
+ try:
+ return max(0.0, float(value))
+ except (TypeError, ValueError):
+ return None
+
+
+def call_zep_read_with_retry(
+ operation: Callable[[], T],
+ *,
+ operation_name: str,
+ max_attempts: int = 3,
+ initial_delay: float = 2.0,
+ max_delay: float = 60.0,
+ sleep: Callable[[float], None] = time.sleep,
+) -> T:
+ """Retry a safe Zep read only for transport, 408, 429, or 5xx errors."""
+
+ if max_attempts < 1:
+ raise ValueError("max_attempts must be at least 1")
+
+ for attempt in range(1, max_attempts + 1):
+ try:
+ return operation()
+ except Exception as error:
+ if attempt == max_attempts or not is_retryable_zep_error(error):
+ raise
+
+ retry_after = _retry_after_seconds(error)
+ delay = min(
+ retry_after if retry_after is not None else initial_delay * (2 ** (attempt - 1)),
+ max_delay,
+ )
+ logger.warning(
+ "Zep %s attempt %s/%s failed (%s); retrying in %.1fs",
+ operation_name,
+ attempt,
+ max_attempts,
+ type(error).__name__,
+ delay,
+ )
+ sleep(delay)
+
+ raise AssertionError("unreachable")
diff --git a/backend/app/utils/zep_lifecycle.py b/backend/app/utils/zep_lifecycle.py
new file mode 100644
index 00000000..34171fe0
--- /dev/null
+++ b/backend/app/utils/zep_lifecycle.py
@@ -0,0 +1,51 @@
+"""Process-local lifecycle coordination for Zep Cloud graphs.
+
+The lock is intentionally keyed by graph ID so graph deletion/reset and a new
+simulation updater claim cannot pass each other between validation and their
+Cloud mutation. It complements (but does not replace) a distributed lock in
+multi-worker deployments.
+"""
+
+import threading
+
+
+_graph_locks: dict[str, threading.RLock] = {}
+_graph_locks_guard = threading.Lock()
+_graph_readers: dict[str, set[str]] = {}
+
+
+def graph_lifecycle_lock(graph_id: str) -> threading.RLock:
+ """Return the process-local re-entrant lifecycle lock for ``graph_id``."""
+
+ if not graph_id:
+ raise ValueError("graph_id is required for lifecycle locking")
+ with _graph_locks_guard:
+ return _graph_locks.setdefault(graph_id, threading.RLock())
+
+
+def register_graph_reader(graph_id: str, reader_id: str) -> None:
+ """Register a long-running read lease under the graph lifecycle lock."""
+
+ if not reader_id:
+ raise ValueError("reader_id is required")
+ with graph_lifecycle_lock(graph_id):
+ _graph_readers.setdefault(graph_id, set()).add(reader_id)
+
+
+def unregister_graph_reader(graph_id: str, reader_id: str) -> None:
+ """Release a previously registered graph read lease."""
+
+ with graph_lifecycle_lock(graph_id):
+ readers = _graph_readers.get(graph_id)
+ if not readers:
+ return
+ readers.discard(reader_id)
+ if not readers:
+ _graph_readers.pop(graph_id, None)
+
+
+def get_graph_readers(graph_id: str) -> list[str]:
+ """Return active reader IDs while serializing with lifecycle mutations."""
+
+ with graph_lifecycle_lock(graph_id):
+ return sorted(_graph_readers.get(graph_id, set()))
diff --git a/backend/app/utils/zep_paging.py b/backend/app/utils/zep_paging.py
index 22c20526..71f4ea58 100644
--- a/backend/app/utils/zep_paging.py
+++ b/backend/app/utils/zep_paging.py
@@ -1,149 +1,160 @@
-"""Zep Graph 分页读取工具。
-
-Zep 的 node/edge 列表接口使用 UUID cursor 分页,
-本模块封装自动翻页逻辑(含单页重试),对调用方透明地返回完整列表。
-"""
+"""Complete Zep Graph node/edge pagination using opaque response cursors."""
from __future__ import annotations
-import time
from collections.abc import Callable
from typing import Any
-from zep_cloud import InternalServerError
from zep_cloud.client import Zep
from .logger import get_logger
+from .zep import call_zep_read_with_retry
-logger = get_logger('mirofish.zep_paging')
+logger = get_logger("mirofish.zep_paging")
_DEFAULT_PAGE_SIZE = 100
-_MAX_NODES = 2000
-_MAX_EDGES = 5000
_DEFAULT_MAX_RETRIES = 3
-_DEFAULT_RETRY_DELAY = 2.0 # seconds, doubles each retry
+_DEFAULT_RETRY_DELAY = 2.0
+_NEXT_CURSOR_HEADER = "zep-next-cursor"
def _fetch_page_with_retry(
- api_call: Callable[..., list[Any]],
+ api_call: Callable[..., Any],
*args: Any,
max_retries: int = _DEFAULT_MAX_RETRIES,
retry_delay: float = _DEFAULT_RETRY_DELAY,
page_description: str = "page",
**kwargs: Any,
-) -> list[Any]:
- """单页请求,失败时指数退避重试。仅重试网络/IO类瞬态错误。"""
- if max_retries < 1:
- raise ValueError("max_retries must be >= 1")
-
- last_exception: Exception | None = None
- delay = retry_delay
-
- for attempt in range(max_retries):
- try:
- return api_call(*args, **kwargs)
- except (ConnectionError, TimeoutError, OSError, InternalServerError) as e:
- last_exception = e
- if attempt < max_retries - 1:
- logger.warning(
- f"Zep {page_description} attempt {attempt + 1} failed: {str(e)[:100]}, retrying in {delay:.1f}s..."
- )
- time.sleep(delay)
- delay *= 2
- else:
- logger.error(f"Zep {page_description} failed after {max_retries} attempts: {str(e)}")
-
- assert last_exception is not None
- raise last_exception
-
-
-def fetch_all_nodes(
- client: Zep,
+) -> Any:
+ """Fetch one read-only page with the shared transient-error policy."""
+
+ return call_zep_read_with_retry(
+ lambda: api_call(*args, **kwargs),
+ operation_name=page_description,
+ max_attempts=max_retries,
+ initial_delay=retry_delay,
+ )
+
+
+def _header_value(headers: Any, name: str) -> str | None:
+ if not headers:
+ return None
+ direct = headers.get(name)
+ if direct is not None:
+ return str(direct)
+ return next(
+ (
+ str(value)
+ for header_name, value in headers.items()
+ if str(header_name).lower() == name.lower()
+ ),
+ None,
+ )
+
+
+def _fetch_all(
+ api_call: Callable[..., Any],
graph_id: str,
- page_size: int = _DEFAULT_PAGE_SIZE,
- max_items: int = _MAX_NODES,
- max_retries: int = _DEFAULT_MAX_RETRIES,
- retry_delay: float = _DEFAULT_RETRY_DELAY,
+ *,
+ item_name: str,
+ page_size: int,
+ max_items: int | None,
+ max_retries: int,
+ retry_delay: float,
) -> list[Any]:
- """分页获取图谱节点,最多返回 max_items 条(默认 2000)。每页请求自带重试。"""
- all_nodes: list[Any] = []
+ if not 1 <= page_size <= 100:
+ raise ValueError("page_size must be between 1 and 100")
+ if max_items is not None and max_items < 1:
+ raise ValueError("max_items must be at least 1 when provided")
+
+ all_items: list[Any] = []
cursor: str | None = None
- page_num = 0
+ seen_cursors: set[str] = set()
+ page_number = 0
while True:
kwargs: dict[str, Any] = {"limit": page_size}
if cursor is not None:
- kwargs["uuid_cursor"] = cursor
+ kwargs["cursor"] = cursor
- page_num += 1
- batch = _fetch_page_with_retry(
- client.graph.node.get_by_graph_id,
+ page_number += 1
+ response = _fetch_page_with_retry(
+ api_call,
graph_id,
max_retries=max_retries,
retry_delay=retry_delay,
- page_description=f"fetch nodes page {page_num} (graph={graph_id})",
+ page_description=(
+ f"fetch {item_name} page {page_number} (graph={graph_id})"
+ ),
**kwargs,
)
- if not batch:
+ batch = list(getattr(response, "data", None) or [])
+ all_items.extend(batch)
+
+ if max_items is not None and len(all_items) >= max_items:
+ if len(all_items) > max_items:
+ all_items = all_items[:max_items]
+ logger.warning(
+ "Zep %s pagination reached explicit max_items=%s for graph %s",
+ item_name,
+ max_items,
+ graph_id,
+ )
break
- all_nodes.extend(batch)
- if len(all_nodes) >= max_items:
- all_nodes = all_nodes[:max_items]
- logger.warning(f"Node count reached limit ({max_items}), stopping pagination for graph {graph_id}")
- break
- if len(batch) < page_size:
- break
-
- cursor = getattr(batch[-1], "uuid_", None) or getattr(batch[-1], "uuid", None)
- if cursor is None:
- logger.warning(f"Node missing uuid field, stopping pagination at {len(all_nodes)} nodes")
+ next_cursor = _header_value(
+ getattr(response, "headers", None),
+ _NEXT_CURSOR_HEADER,
+ )
+ if next_cursor is None:
break
+ if next_cursor in seen_cursors or next_cursor == cursor:
+ raise RuntimeError(
+ f"Zep {item_name} pagination cursor did not advance for graph {graph_id}"
+ )
+ seen_cursors.add(next_cursor)
+ cursor = next_cursor
- return all_nodes
+ return all_items
-def fetch_all_edges(
+def fetch_all_nodes(
client: Zep,
graph_id: str,
page_size: int = _DEFAULT_PAGE_SIZE,
+ max_items: int | None = None,
max_retries: int = _DEFAULT_MAX_RETRIES,
retry_delay: float = _DEFAULT_RETRY_DELAY,
- max_items: int = _MAX_EDGES,
) -> list[Any]:
- """分页获取图谱所有边,最多返回 max_items 条(默认 5000)。每页请求自带重试。"""
- all_edges: list[Any] = []
- cursor: str | None = None
- page_num = 0
-
- while True:
- kwargs: dict[str, Any] = {"limit": page_size}
- if cursor is not None:
- kwargs["uuid_cursor"] = cursor
+ """Fetch every graph node unless the caller supplies an explicit cap."""
- page_num += 1
- batch = _fetch_page_with_retry(
- client.graph.edge.get_by_graph_id,
- graph_id,
- max_retries=max_retries,
- retry_delay=retry_delay,
- page_description=f"fetch edges page {page_num} (graph={graph_id})",
- **kwargs,
- )
- if not batch:
- break
+ return _fetch_all(
+ client.graph.node.with_raw_response.get_by_graph_id,
+ graph_id,
+ item_name="nodes",
+ page_size=page_size,
+ max_items=max_items,
+ max_retries=max_retries,
+ retry_delay=retry_delay,
+ )
- all_edges.extend(batch)
- if len(all_edges) >= max_items:
- all_edges = all_edges[:max_items]
- logger.warning(f"Edge count reached limit ({max_items}), stopping pagination for graph {graph_id}")
- break
- if len(batch) < page_size:
- break
- cursor = getattr(batch[-1], "uuid_", None) or getattr(batch[-1], "uuid", None)
- if cursor is None:
- logger.warning(f"Edge missing uuid field, stopping pagination at {len(all_edges)} edges")
- break
-
- return all_edges
+def fetch_all_edges(
+ client: Zep,
+ graph_id: str,
+ page_size: int = _DEFAULT_PAGE_SIZE,
+ max_retries: int = _DEFAULT_MAX_RETRIES,
+ retry_delay: float = _DEFAULT_RETRY_DELAY,
+ max_items: int | None = None,
+) -> list[Any]:
+ """Fetch every graph edge unless the caller supplies an explicit cap."""
+
+ return _fetch_all(
+ client.graph.edge.with_raw_response.get_by_graph_id,
+ graph_id,
+ item_name="edges",
+ page_size=page_size,
+ max_items=max_items,
+ max_retries=max_retries,
+ retry_delay=retry_delay,
+ )
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 8c65b729..f972e120 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -17,7 +17,8 @@ dependencies = [
"openai>=1.0.0",
# Zep Cloud
- "zep-cloud==3.13.0",
+ "zep-cloud==3.25.0",
+ "httpx>=0.27.0",
# OASIS 社交媒体模拟
"camel-oasis==0.2.5",
diff --git a/backend/requirements.txt b/backend/requirements.txt
index 4f146296..8fe1327e 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -14,7 +14,8 @@ flask-cors>=6.0.0
openai>=1.0.0
# ============= Zep Cloud =============
-zep-cloud==3.13.0
+zep-cloud==3.25.0
+httpx>=0.27.0
# ============= OASIS 社交媒体模拟 =============
# OASIS 社交模拟框架
diff --git a/backend/tests/test_ontology_attributes.py b/backend/tests/test_ontology_attributes.py
index 38e96546..bbc7bd35 100644
--- a/backend/tests/test_ontology_attributes.py
+++ b/backend/tests/test_ontology_attributes.py
@@ -2,6 +2,7 @@ from app.services.ontology_generator import OntologyGenerator
from app.services.graph_builder import GraphBuilderService
from app.utils.ontology import (
MAX_ONTOLOGY_ATTRIBUTES,
+ MAX_ONTOLOGY_SOURCE_TARGETS,
normalize_ontology_attribute,
normalize_ontology_attributes,
)
@@ -168,3 +169,89 @@ def test_graph_builder_passes_an_empty_entity_mapping_for_edge_only_ontology():
})
assert captured["entities"] == {}
+
+
+def test_graph_builder_deduplicates_and_caps_edge_source_targets_for_zep():
+ captured = {}
+
+ class GraphApi:
+ def set_ontology(self, **kwargs):
+ captured.update(kwargs)
+
+ class Client:
+ graph = GraphApi()
+
+ source_targets = [
+ {"source": f"Source{index}", "target": f"Target{index}"}
+ for index in range(MAX_ONTOLOGY_SOURCE_TARGETS + 2)
+ ]
+ source_targets.insert(1, dict(source_targets[0]))
+
+ builder = object.__new__(GraphBuilderService)
+ builder.client = Client()
+ builder.set_ontology("graph-id", {
+ "entity_types": [],
+ "edge_types": [{
+ "name": "RELATED_TO",
+ "attributes": ["reason"],
+ "source_targets": source_targets,
+ }],
+ })
+
+ _, normalized_targets = captured["edges"]["RELATED_TO"]
+ assert len(normalized_targets) == MAX_ONTOLOGY_SOURCE_TARGETS
+ assert [(item.source, item.target) for item in normalized_targets] == [
+ (f"Source{index}", f"Target{index}")
+ for index in range(MAX_ONTOLOGY_SOURCE_TARGETS)
+ ]
+
+
+def test_generator_ignores_invalid_entries_and_normalizes_edge_names():
+ source_targets = [
+ {"source": "speaker", "target": "news outlet"},
+ {"source": "speaker", "target": "news outlet"},
+ None,
+ ] + [
+ {"source": "speaker", "target": "news outlet" if index == 0 else "Person"}
+ for index in range(12)
+ ]
+
+ result = OntologyGenerator(llm_client=object())._validate_and_process({
+ "entity_types": ["speaker", None, 7, {"name": "news outlet"}],
+ "edge_types": [
+ "unusable edge",
+ None,
+ {"name": "worksFor", "source_targets": source_targets},
+ {"name": "works-for", "source_targets": []},
+ ],
+ })
+
+ assert [entity["name"] for entity in result["entity_types"][:2]] == [
+ "Speaker",
+ "NewsOutlet",
+ ]
+ assert [edge["name"] for edge in result["edge_types"]] == ["WORKS_FOR"]
+ assert result["edge_types"][0]["source_targets"] == [
+ {"source": "Speaker", "target": "NewsOutlet"},
+ {"source": "Speaker", "target": "Person"},
+ ]
+
+
+def test_generator_caps_after_discarding_invalid_edge_endpoints():
+ invalid_first = [
+ {"source": f"Removed{index}", "target": "AlsoRemoved"}
+ for index in range(MAX_ONTOLOGY_SOURCE_TARGETS)
+ ]
+ result = OntologyGenerator(llm_client=object())._validate_and_process({
+ "entity_types": [{"name": "person"}, {"name": "organization"}],
+ "edge_types": [{
+ "name": "works_for",
+ "source_targets": invalid_first + [
+ {"source": "person", "target": "organization"}
+ ],
+ }],
+ })
+
+ assert result["edge_types"][0]["source_targets"] == [
+ {"source": "Person", "target": "Organization"}
+ ]
diff --git a/backend/tests/test_zep_cloud_contracts.py b/backend/tests/test_zep_cloud_contracts.py
new file mode 100644
index 00000000..0d0bad40
--- /dev/null
+++ b/backend/tests/test_zep_cloud_contracts.py
@@ -0,0 +1,437 @@
+from types import SimpleNamespace
+import json
+
+import httpx
+import pytest
+from zep_cloud import Zep
+from zep_cloud.core.api_error import ApiError as ZepApiError
+
+from app.services import graph_builder as graph_builder_module
+from app.services.graph_builder import BatchSubmission, GraphBuilderService
+from app.services.oasis_profile_generator import OasisProfileGenerator
+from app.services.zep_entity_reader import EntityNode, ZepEntityReader
+from app.services.zep_tools import ZepToolsService
+
+
+def test_report_search_caps_the_query_sent_to_zep():
+ calls = []
+
+ class GraphApi:
+ def search(self, **kwargs):
+ calls.append(kwargs)
+ return SimpleNamespace(edges=[], nodes=[])
+
+ service = object.__new__(ZepToolsService)
+ service.client = SimpleNamespace(graph=GraphApi())
+
+ original_query = "q" * 401
+ result = service.search_graph("graph-id", original_query)
+
+ assert calls[0]["query"] == original_query[:400]
+ assert result.query == original_query
+
+
+def test_profile_context_search_caps_both_queries_sent_to_zep():
+ calls = []
+
+ class GraphApi:
+ def search(self, **kwargs):
+ calls.append(kwargs)
+ return SimpleNamespace(edges=[], nodes=[])
+
+ generator = object.__new__(OasisProfileGenerator)
+ generator.zep_client = SimpleNamespace(graph=GraphApi())
+ generator.graph_id = "graph-id"
+
+ entity = EntityNode(
+ uuid="node-id",
+ name="n" * 500,
+ labels=["Entity", "Person"],
+ summary="",
+ attributes={},
+ )
+ generator._search_zep_for_entity(entity)
+
+ assert len(calls) == 2
+ assert all(0 < len(call["query"]) <= 400 for call in calls)
+
+
+def test_entity_reader_does_not_turn_auth_failure_into_missing_entity():
+ def unauthorized(**_kwargs):
+ raise ZepApiError(status_code=401, body={"message": "unauthorized"})
+
+ reader = object.__new__(ZepEntityReader)
+ reader.client = SimpleNamespace(
+ graph=SimpleNamespace(node=SimpleNamespace(get=unauthorized))
+ )
+
+ with pytest.raises(ZepApiError) as error:
+ reader.get_entity_with_context("graph-id", "node-id")
+
+ assert error.value.status_code == 401
+
+
+def test_entity_reader_does_not_turn_edge_failure_into_empty_data():
+ def forbidden(**_kwargs):
+ raise ZepApiError(status_code=403, body={"message": "forbidden"})
+
+ reader = object.__new__(ZepEntityReader)
+ reader.client = SimpleNamespace(
+ graph=SimpleNamespace(
+ node=SimpleNamespace(get_edges=forbidden),
+ )
+ )
+
+ with pytest.raises(ZepApiError) as error:
+ reader.get_node_edges("node-id")
+
+ assert error.value.status_code == 403
+
+
+def test_report_tools_do_not_turn_zep_read_failures_into_empty_data():
+ def unauthorized(**_kwargs):
+ raise ZepApiError(status_code=401, body={"message": "unauthorized"})
+
+ service = object.__new__(ZepToolsService)
+ service.client = SimpleNamespace(
+ graph=SimpleNamespace(node=SimpleNamespace(get=unauthorized))
+ )
+
+ with pytest.raises(ZepApiError):
+ service.get_node_detail("node-id")
+
+ service.get_all_edges = lambda _graph_id: (_ for _ in ()).throw(
+ ZepApiError(status_code=503, body={"message": "unavailable"})
+ )
+ with pytest.raises(ZepApiError):
+ service.get_node_edges("graph-id", "node-id")
+
+
+def test_episode_processing_timeout_fails_instead_of_reporting_success(monkeypatch):
+ builder = object.__new__(GraphBuilderService)
+ builder.client = SimpleNamespace(
+ graph=SimpleNamespace(
+ episode=SimpleNamespace(
+ get=lambda **_kwargs: SimpleNamespace(processed=False)
+ )
+ )
+ )
+
+ timestamps = iter([0.0, 2.0])
+ monkeypatch.setattr(graph_builder_module.time, "time", lambda: next(timestamps))
+ monkeypatch.setattr(graph_builder_module.time, "sleep", lambda _seconds: None)
+
+ with pytest.raises(TimeoutError, match="episode"):
+ builder._wait_for_episodes(["episode-1"], timeout=1)
+
+
+def test_document_ingestion_uses_current_batch_api_and_persists_identity():
+ calls = []
+
+ class BatchApi:
+ def create(self, **kwargs):
+ calls.append(("create", kwargs))
+ return SimpleNamespace(batch_id="batch-1")
+
+ def add(self, **kwargs):
+ calls.append(("add", kwargs))
+ return [
+ SimpleNamespace(episode_uuid=f"episode-{index}")
+ for index, _item in enumerate(kwargs["items"])
+ ]
+
+ def process(self, **kwargs):
+ calls.append(("process", kwargs))
+ return SimpleNamespace(status="queued")
+
+ builder = object.__new__(GraphBuilderService)
+ builder.client = SimpleNamespace(batch=BatchApi())
+ persisted = []
+
+ submission = builder.add_text_batches(
+ "graph-id",
+ ["chunk one", "chunk two"],
+ batch_created_callback=lambda batch_id, operation_id: persisted.append(
+ (batch_id, operation_id)
+ ),
+ )
+
+ assert submission.batch_id == "batch-1"
+ assert submission.item_count == 2
+ assert len(submission.operation_id) == 64
+ assert persisted == [
+ (None, submission.operation_id),
+ ("batch-1", submission.operation_id),
+ ]
+ assert [name for name, _kwargs in calls] == ["create", "add", "process"]
+ items = calls[1][1]["items"]
+ assert [item.type for item in items] == ["graph_episode", "graph_episode"]
+ assert all(item.graph_id == "graph-id" for item in items)
+ assert all(item.data_type == "text" for item in items)
+
+
+def test_graph_create_persists_identity_before_post_and_reconciles_timeout():
+ events = []
+
+ class GraphApi:
+ def create(self, **kwargs):
+ events.append(("create", kwargs["graph_id"]))
+ raise TimeoutError("response lost")
+
+ def get(self, graph_id):
+ events.append(("get", graph_id))
+ return SimpleNamespace(graph_id=graph_id)
+
+ builder = object.__new__(GraphBuilderService)
+ builder.client = SimpleNamespace(graph=GraphApi())
+
+ graph_id = builder.create_graph(
+ "Graph",
+ graph_id="known-id",
+ graph_id_callback=lambda value: events.append(("persist", value)),
+ )
+
+ assert graph_id == "known-id"
+ assert events == [
+ ("persist", "known-id"),
+ ("create", "known-id"),
+ ("get", "known-id"),
+ ]
+
+
+def test_batch_create_timeout_is_reconciled_by_operation_metadata(monkeypatch):
+ calls = []
+ list_count = 0
+
+ class BatchApi:
+ def create(self, **_kwargs):
+ calls.append("create")
+ raise TimeoutError("response lost")
+
+ def list(self, **_kwargs):
+ nonlocal list_count
+ calls.append("list")
+ list_count += 1
+ if list_count == 1:
+ return SimpleNamespace(batches=[], next_cursor=None)
+ return SimpleNamespace(
+ batches=[SimpleNamespace(
+ batch_id="batch-recovered",
+ metadata={
+ "mirofish_operation_id": GraphBuilderService.build_operation_id(
+ "graph-id", ["chunk"]
+ ),
+ "graph_id": "graph-id",
+ },
+ )],
+ next_cursor=None,
+ )
+
+ def add(self, **kwargs):
+ calls.append("add")
+ return [SimpleNamespace(episode_uuid="episode-1")]
+
+ def process(self, **_kwargs):
+ calls.append("process")
+ return SimpleNamespace(status="queued")
+
+ builder = object.__new__(GraphBuilderService)
+ builder.client = SimpleNamespace(batch=BatchApi())
+ monkeypatch.setattr(graph_builder_module.time, "sleep", lambda _seconds: None)
+
+ submission = builder.add_text_batches("graph-id", ["chunk"])
+
+ assert submission.batch_id == "batch-recovered"
+ assert calls == ["create", "list", "list", "add", "process"]
+
+
+def test_batch_add_timeout_recovers_a_fully_accepted_group_without_replay(monkeypatch):
+ add_calls = []
+ list_calls = []
+
+ class BatchApi:
+ def create(self, **_kwargs):
+ return SimpleNamespace(batch_id="batch-1")
+
+ def add(self, **_kwargs):
+ add_calls.append(True)
+ raise TimeoutError("response lost")
+
+ def list_items(self, **_kwargs):
+ list_calls.append(True)
+ if len(list_calls) == 1:
+ return SimpleNamespace(items=[], next_cursor=None)
+ return SimpleNamespace(
+ items=[
+ SimpleNamespace(sequence_index=0, episode_uuid="episode-1"),
+ SimpleNamespace(sequence_index=1, episode_uuid="episode-2"),
+ ],
+ next_cursor=None,
+ )
+
+ def process(self, **_kwargs):
+ return SimpleNamespace(status="queued")
+
+ builder = object.__new__(GraphBuilderService)
+ builder.client = SimpleNamespace(batch=BatchApi())
+ monkeypatch.setattr(graph_builder_module.time, "sleep", lambda _seconds: None)
+
+ submission = builder.add_text_batches(
+ "graph-id", ["chunk one", "chunk two"]
+ )
+
+ assert submission.item_count == 2
+ assert add_calls == [True]
+ assert len(list_calls) == 2
+
+
+def test_batch_wait_validates_terminal_items_and_opaque_zero_cursor():
+ list_calls = []
+
+ class BatchApi:
+ def get(self, **_kwargs):
+ return SimpleNamespace(
+ status="succeeded",
+ progress=SimpleNamespace(
+ percent_complete=100,
+ succeeded_items=2,
+ ),
+ )
+
+ def list_items(self, **kwargs):
+ list_calls.append(kwargs)
+ if kwargs["cursor"] is None:
+ return SimpleNamespace(
+ items=[SimpleNamespace(
+ sequence_index=0,
+ status="succeeded",
+ episode_uuid="episode-1",
+ source_uuid="episode-1",
+ )],
+ next_cursor=0,
+ )
+ return SimpleNamespace(
+ items=[SimpleNamespace(
+ sequence_index=1,
+ status="succeeded",
+ episode_uuid="episode-2",
+ source_uuid="episode-2",
+ )],
+ next_cursor=None,
+ )
+
+ builder = object.__new__(GraphBuilderService)
+ builder.client = SimpleNamespace(batch=BatchApi())
+ submission = BatchSubmission("batch-1", "operation", [], 2)
+
+ assert builder._wait_for_batch(submission, timeout=1) == [
+ "episode-1",
+ "episode-2",
+ ]
+ assert [call["cursor"] for call in list_calls] == [None, 0]
+
+
[email protected]("status", ["partial", "failed", "invalid", "canceled"])
+def test_batch_non_success_terminal_states_fail(status):
+ builder = object.__new__(GraphBuilderService)
+ builder.client = SimpleNamespace(
+ batch=SimpleNamespace(
+ get=lambda **_kwargs: SimpleNamespace(status=status, progress=None),
+ list_items=lambda **_kwargs: SimpleNamespace(
+ items=[SimpleNamespace(status="failed", error={"message": "bad"})],
+ next_cursor=None,
+ ),
+ )
+ )
+
+ with pytest.raises(RuntimeError, match=status):
+ builder._wait_for_batch(
+ BatchSubmission("batch-1", "operation", [], 1),
+ timeout=1,
+ )
+
+
+def test_batch_wait_times_out_while_status_remains_nonterminal(monkeypatch):
+ builder = object.__new__(GraphBuilderService)
+ builder.client = SimpleNamespace(
+ batch=SimpleNamespace(
+ get=lambda **_kwargs: SimpleNamespace(status="processing", progress=None)
+ )
+ )
+ timestamps = iter([0.0, 2.0])
+ monkeypatch.setattr(graph_builder_module.time, "time", lambda: next(timestamps))
+ monkeypatch.setattr(graph_builder_module.time, "sleep", lambda _seconds: None)
+
+ with pytest.raises(TimeoutError, match="batch-1"):
+ builder._wait_for_batch(
+ BatchSubmission("batch-1", "operation", [], 1),
+ timeout=1,
+ )
+
+
+def test_installed_sdk_serializes_the_batch_325_contract():
+ requests = []
+
+ def handler(request):
+ requests.append((request.method, request.url.path, request.content))
+ path = request.url.path
+ if path.endswith("/batches") and request.method == "POST":
+ return httpx.Response(
+ 200,
+ json={"batch_id": "batch-1", "status": "draft", "item_count": 0},
+ )
+ if path.endswith("/batches/batch-1/items") and request.method == "POST":
+ return httpx.Response(200, json=[{
+ "item_id": "item-1",
+ "sequence_index": 0,
+ "status": "pending",
+ "episode_uuid": "episode-1",
+ "source_uuid": "episode-1",
+ }])
+ if path.endswith("/batches/batch-1/process"):
+ return httpx.Response(
+ 200,
+ json={"batch_id": "batch-1", "status": "queued", "item_count": 1},
+ )
+ if path.endswith("/batches/batch-1"):
+ return httpx.Response(200, json={
+ "batch_id": "batch-1",
+ "status": "succeeded",
+ "item_count": 1,
+ "progress": {"percent_complete": 100, "succeeded_items": 1},
+ })
+ if path.endswith("/batches/batch-1/items") and request.method == "GET":
+ return httpx.Response(200, json={
+ "items": [{
+ "item_id": "item-1",
+ "sequence_index": 0,
+ "status": "succeeded",
+ "episode_uuid": "episode-1",
+ "source_uuid": "episode-1",
+ }],
+ "next_cursor": None,
+ })
+ raise AssertionError(f"Unexpected request: {request.method} {path}")
+
+ with httpx.Client(transport=httpx.MockTransport(handler)) as transport_client:
+ builder = object.__new__(GraphBuilderService)
+ builder.client = Zep(api_key="test-key", httpx_client=transport_client)
+ submission = builder.add_text_batches("graph-id", ["source chunk"])
+ assert builder._wait_for_batch(submission, timeout=1) == ["episode-1"]
+
+ assert [(method, path) for method, path, _body in requests] == [
+ ("POST", "/api/v2/batches"),
+ ("POST", "/api/v2/batches/batch-1/items"),
+ ("POST", "/api/v2/batches/batch-1/process"),
+ ("GET", "/api/v2/batches/batch-1"),
+ ("GET", "/api/v2/batches/batch-1/items"),
+ ]
+ add_payload = json.loads(requests[1][2])
+ assert add_payload["items"][0] == {
+ "data": "source chunk",
+ "data_type": "text",
+ "graph_id": "graph-id",
+ "metadata": add_payload["items"][0]["metadata"],
+ "source_description": "MiroFish source document chunk",
+ "type": "graph_episode",
+ }
diff --git a/backend/tests/test_zep_edge_paging.py b/backend/tests/test_zep_edge_paging.py
index d0e92ff4..d69ed0f1 100644
--- a/backend/tests/test_zep_edge_paging.py
+++ b/backend/tests/test_zep_edge_paging.py
@@ -4,14 +4,27 @@ from app.utils import zep_paging
def _client():
- edge_api = SimpleNamespace(get_by_graph_id=lambda *args, **kwargs: [])
+ edge_api = SimpleNamespace(
+ with_raw_response=SimpleNamespace(
+ get_by_graph_id=lambda *args, **kwargs: SimpleNamespace(
+ data=[],
+ headers={},
+ )
+ )
+ )
return SimpleNamespace(graph=SimpleNamespace(edge=edge_api))
def test_edge_cap_stops_pagination_at_requested_limit(monkeypatch):
pages = [
- [SimpleNamespace(uuid_="e1"), SimpleNamespace(uuid_="e2")],
- [SimpleNamespace(uuid_="e3"), SimpleNamespace(uuid_="e4")],
+ SimpleNamespace(
+ data=[SimpleNamespace(uuid_="e1"), SimpleNamespace(uuid_="e2")],
+ headers={"Zep-Next-Cursor": "opaque-page-2"},
+ ),
+ SimpleNamespace(
+ data=[SimpleNamespace(uuid_="e3"), SimpleNamespace(uuid_="e4")],
+ headers={},
+ ),
]
calls = []
@@ -25,7 +38,7 @@ def test_edge_cap_stops_pagination_at_requested_limit(monkeypatch):
assert [edge.uuid_ for edge in result] == ["e1", "e2", "e3"]
assert len(calls) == 2
- assert calls[1]["uuid_cursor"] == "e2"
+ assert calls[1]["cursor"] == "opaque-page-2"
def test_existing_positional_retry_arguments_keep_their_meaning(monkeypatch):
@@ -33,7 +46,7 @@ def test_existing_positional_retry_arguments_keep_their_meaning(monkeypatch):
def fake_fetch(*args, **kwargs):
observed.update(kwargs)
- return []
+ return SimpleNamespace(data=[], headers={})
monkeypatch.setattr(zep_paging, "_fetch_page_with_retry", fake_fetch)
@@ -42,3 +55,58 @@ def test_existing_positional_retry_arguments_keep_their_meaning(monkeypatch):
assert observed["limit"] == 25
assert observed["max_retries"] == 7
assert observed["retry_delay"] == 0.25
+
+
+def test_pagination_uses_the_current_opaque_response_cursor():
+ calls = []
+
+ class RawEdgeApi:
+ def get_by_graph_id(self, graph_id, **kwargs):
+ calls.append((graph_id, kwargs))
+ if kwargs.get("cursor") is None:
+ return SimpleNamespace(
+ data=[SimpleNamespace(uuid_="e1"), SimpleNamespace(uuid_="e2")],
+ headers={"Zep-Next-Cursor": "opaque-page-2"},
+ )
+ return SimpleNamespace(
+ data=[SimpleNamespace(uuid_="e3")],
+ headers={},
+ )
+
+ class EdgeApi:
+ with_raw_response = RawEdgeApi()
+
+ def get_by_graph_id(self, *_args, **_kwargs):
+ raise AssertionError("pagination must read the response cursor header")
+
+ client = SimpleNamespace(
+ graph=SimpleNamespace(edge=EdgeApi())
+ )
+
+ result = zep_paging.fetch_all_edges(client, "graph", page_size=2)
+
+ assert [edge.uuid_ for edge in result] == ["e1", "e2", "e3"]
+ assert calls == [
+ ("graph", {"limit": 2}),
+ ("graph", {"limit": 2, "cursor": "opaque-page-2"}),
+ ]
+
+
+def test_pagination_fails_if_the_service_repeats_a_cursor():
+ class RawEdgeApi:
+ def get_by_graph_id(self, _graph_id, **_kwargs):
+ return SimpleNamespace(
+ data=[SimpleNamespace(uuid_="e1")],
+ headers={"Zep-Next-Cursor": "same-cursor"},
+ )
+
+ client = SimpleNamespace(
+ graph=SimpleNamespace(
+ edge=SimpleNamespace(with_raw_response=RawEdgeApi())
+ )
+ )
+
+ import pytest
+
+ with pytest.raises(RuntimeError, match="did not advance"):
+ zep_paging.fetch_all_edges(client, "graph", page_size=1)
diff --git a/backend/tests/test_zep_graph_lifecycle.py b/backend/tests/test_zep_graph_lifecycle.py
new file mode 100644
index 00000000..947ec0f2
--- /dev/null
+++ b/backend/tests/test_zep_graph_lifecycle.py
@@ -0,0 +1,418 @@
+from datetime import datetime
+import threading
+
+from flask import Flask
+from types import SimpleNamespace
+
+from app.api import graph as graph_api
+from app.api import simulation as simulation_api
+from app.models.project import Project, ProjectStatus
+from app.services.simulation_manager import SimulationStatus
+from app.models.task import TaskStatus
+
+
+def _project(status, graph_id="graph-1"):
+ now = datetime.now().isoformat()
+ return Project(
+ project_id="proj-1",
+ name="Project",
+ status=status,
+ created_at=now,
+ updated_at=now,
+ ontology={"entity_types": [], "edge_types": []},
+ graph_id=graph_id,
+ graph_build_task_id="task-1",
+ zep_batch_id="batch-1",
+ zep_batch_operation_id="operation-1",
+ )
+
+
+def _json_result(result):
+ if isinstance(result, tuple):
+ response, status = result
+ else:
+ response, status = result, result.status_code
+ return response.get_json(), status
+
+
+def test_project_reset_deletes_the_cloud_graph_before_clearing_reference(monkeypatch):
+ project = _project(ProjectStatus.GRAPH_COMPLETED)
+ events = []
+
+ class Builder:
+ def __init__(self, **_kwargs):
+ pass
+
+ def delete_graph(self, graph_id):
+ events.append(("cloud-delete", graph_id))
+
+ monkeypatch.setattr(graph_api, "GraphBuilderService", Builder)
+ monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "get_project",
+ classmethod(lambda _cls, _project_id: project),
+ )
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "save_project",
+ classmethod(lambda _cls, saved: events.append(("save", saved.graph_id))),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context("/api/graph/project/proj-1/reset", method="POST"):
+ body, status = _json_result(graph_api.reset_project("proj-1"))
+
+ assert status == 200
+ assert body["success"] is True
+ assert events == [("cloud-delete", "graph-1"), ("save", None)]
+ assert project.zep_batch_id is None
+ assert project.status == ProjectStatus.ONTOLOGY_GENERATED
+
+
+def test_project_reset_refuses_a_graph_with_an_active_simulation(monkeypatch):
+ project = _project(ProjectStatus.GRAPH_COMPLETED)
+ monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "get_project",
+ classmethod(lambda _cls, _project_id: project),
+ )
+ monkeypatch.setattr(
+ graph_api.ZepGraphMemoryManager,
+ "get_simulation_ids_for_graph",
+ classmethod(lambda _cls, _graph_id: ["sim-active"]),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context("/api/graph/project/proj-1/reset", method="POST"):
+ body, status = _json_result(graph_api.reset_project("proj-1"))
+
+ assert status == 409
+ assert "sim-active" in body["error"]
+
+
+def test_graph_delete_cannot_discard_an_updater_during_finalization(monkeypatch):
+ monkeypatch.setattr(
+ graph_api.ZepGraphMemoryManager,
+ "get_simulation_ids_for_graph",
+ classmethod(lambda _cls, _graph_id: ["sim-finalizing"]),
+ )
+ discarded = []
+ monkeypatch.setattr(
+ graph_api.ZepGraphMemoryManager,
+ "discard_inactive_updater",
+ classmethod(
+ lambda _cls, simulation_id: discarded.append(simulation_id)
+ ),
+ )
+ lock = graph_api.SimulationRunner._finalization_lock("sim-finalizing")
+ lock.acquire()
+ try:
+ assert graph_api._active_graph_consumers("graph-1") == ["sim-finalizing"]
+ finally:
+ lock.release()
+
+ assert discarded == []
+
+
+def test_repeated_build_request_reuses_the_existing_task(monkeypatch):
+ project = _project(ProjectStatus.GRAPH_BUILDING)
+ monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "get_project",
+ classmethod(lambda _cls, _project_id: project),
+ )
+ monkeypatch.setattr(
+ graph_api,
+ "TaskManager",
+ lambda: SimpleNamespace(
+ get_task=lambda _task_id: SimpleNamespace(status=TaskStatus.PROCESSING)
+ ),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context(
+ "/api/graph/build",
+ method="POST",
+ json={"project_id": "proj-1", "force": True},
+ ):
+ body, status = _json_result(graph_api.build_graph())
+
+ assert status == 200
+ assert body["success"] is True
+ assert body["data"]["reused"] is True
+ assert body["data"]["task_id"] == "task-1"
+ assert body["data"]["graph_id"] == "graph-1"
+
+
+def test_stale_build_after_restart_is_recoverable_instead_of_reused(monkeypatch):
+ project = _project(ProjectStatus.GRAPH_BUILDING)
+ project.zep_batch_id = None
+ project.zep_batch_operation_id = None
+ saved = []
+ monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "get_project",
+ classmethod(lambda _cls, _project_id: project),
+ )
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "save_project",
+ classmethod(lambda _cls, value: saved.append(value.status)),
+ )
+ monkeypatch.setattr(
+ graph_api,
+ "TaskManager",
+ lambda: SimpleNamespace(get_task=lambda _task_id: None),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context(
+ "/api/graph/build",
+ method="POST",
+ json={"project_id": "proj-1"},
+ ):
+ body, status = _json_result(graph_api.build_graph())
+
+ assert status == 409
+ assert body["recoverable"] is True
+ assert project.status == ProjectStatus.FAILED
+ assert saved == [ProjectStatus.FAILED]
+
+
+def test_stale_build_resumes_a_persisted_processing_batch(monkeypatch):
+ project = _project(ProjectStatus.GRAPH_BUILDING)
+ created_threads = []
+
+ class Tasks:
+ def get_task(self, _task_id):
+ return None
+
+ def create_task(self, _description):
+ return "task-resumed"
+
+ class Builder:
+ def __init__(self, **_kwargs):
+ pass
+
+ def get_batch_summary(self, batch_id):
+ assert batch_id == "batch-1"
+ return SimpleNamespace(status="processing")
+
+ class Thread:
+ def __init__(self, *, target, daemon):
+ created_threads.append((target, daemon))
+
+ def start(self):
+ pass
+
+ monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
+ monkeypatch.setattr(graph_api, "TaskManager", Tasks)
+ monkeypatch.setattr(graph_api, "GraphBuilderService", Builder)
+ monkeypatch.setattr(graph_api.threading, "Thread", Thread)
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "get_project",
+ classmethod(lambda _cls, _project_id: project),
+ )
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "get_extracted_text",
+ classmethod(lambda _cls, _project_id: "source text"),
+ )
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "save_project",
+ classmethod(lambda _cls, _project: None),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context(
+ "/api/graph/build",
+ method="POST",
+ json={"project_id": "proj-1"},
+ ):
+ body, status = _json_result(graph_api.build_graph())
+
+ assert status == 200
+ assert body["data"]["resumed"] is True
+ assert body["data"]["task_id"] == "task-resumed"
+ assert project.graph_build_task_id == "task-resumed"
+ assert len(created_threads) == 1
+
+
+def test_project_delete_removes_cloud_graph_before_local_files(monkeypatch):
+ project = _project(ProjectStatus.GRAPH_COMPLETED)
+ events = []
+
+ class Builder:
+ def __init__(self, **_kwargs):
+ pass
+
+ def delete_graph(self, graph_id):
+ events.append(("cloud-delete", graph_id))
+
+ monkeypatch.setattr(graph_api, "GraphBuilderService", Builder)
+ monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "get_project",
+ classmethod(lambda _cls, _project_id: project),
+ )
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "delete_project",
+ classmethod(
+ lambda _cls, project_id: events.append(("local-delete", project_id)) or True
+ ),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context(
+ "/api/graph/project/proj-1",
+ method="DELETE",
+ ):
+ body, status = _json_result(graph_api.delete_project("proj-1"))
+
+ assert status == 200
+ assert body["success"] is True
+ assert events == [
+ ("cloud-delete", "graph-1"),
+ ("local-delete", "proj-1"),
+ ]
+
+
+def test_completed_build_request_is_idempotent_without_force(monkeypatch):
+ project = _project(ProjectStatus.GRAPH_COMPLETED)
+ monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "get_project",
+ classmethod(lambda _cls, _project_id: project),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context(
+ "/api/graph/build",
+ method="POST",
+ json={"project_id": "proj-1"},
+ ):
+ body, status = _json_result(graph_api.build_graph())
+
+ assert status == 200
+ assert body["data"]["reused"] is True
+ assert body["data"]["graph_id"] == "graph-1"
+
+
+def test_force_must_be_a_json_boolean(monkeypatch):
+ project = _project(ProjectStatus.GRAPH_COMPLETED)
+ monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "get_project",
+ classmethod(lambda _cls, _project_id: project),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context(
+ "/api/graph/build",
+ method="POST",
+ json={"project_id": "proj-1", "force": "false"},
+ ):
+ body, status = _json_result(graph_api.build_graph())
+
+ assert status == 400
+ assert "boolean" in body["error"]
+
+
+def test_graph_reset_and_memory_start_cannot_cross_between_delete_and_clear(
+ monkeypatch,
+):
+ project = _project(ProjectStatus.GRAPH_COMPLETED)
+ simulation = SimpleNamespace(
+ simulation_id="sim-1",
+ project_id=project.project_id,
+ graph_id=project.graph_id,
+ status=SimulationStatus.READY,
+ )
+ delete_entered = threading.Event()
+ allow_delete = threading.Event()
+ runner_called = []
+
+ class Builder:
+ def __init__(self, **_kwargs):
+ pass
+
+ def delete_graph(self, graph_id):
+ assert graph_id == "graph-1"
+ delete_entered.set()
+ assert allow_delete.wait(timeout=2)
+
+ class Simulations:
+ def get_simulation(self, _simulation_id):
+ return simulation
+
+ monkeypatch.setattr(graph_api, "GraphBuilderService", Builder)
+ monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "get_project",
+ classmethod(lambda _cls, _project_id: project),
+ )
+ monkeypatch.setattr(
+ graph_api.ProjectManager,
+ "save_project",
+ classmethod(lambda _cls, _project: None),
+ )
+ monkeypatch.setattr(simulation_api, "SimulationManager", Simulations)
+ monkeypatch.setattr(
+ simulation_api.ProjectManager,
+ "get_project",
+ classmethod(lambda _cls, _project_id: project),
+ )
+ monkeypatch.setattr(
+ simulation_api.SimulationRunner,
+ "start_simulation",
+ classmethod(lambda _cls, **_kwargs: runner_called.append(True)),
+ )
+
+ app = Flask(__name__)
+ results = {}
+
+ def reset():
+ with app.test_request_context(
+ "/api/graph/project/proj-1/reset", method="POST"
+ ):
+ results["reset"] = _json_result(graph_api.reset_project("proj-1"))
+
+ def start():
+ with app.test_request_context(
+ "/api/simulation/start",
+ method="POST",
+ json={
+ "simulation_id": "sim-1",
+ "enable_graph_memory_update": True,
+ },
+ ):
+ results["start"] = _json_result(simulation_api.start_simulation())
+
+ reset_thread = threading.Thread(target=reset)
+ reset_thread.start()
+ assert delete_entered.wait(timeout=2)
+
+ start_thread = threading.Thread(target=start)
+ start_thread.start()
+ start_thread.join(timeout=0.05)
+ assert start_thread.is_alive()
+
+ allow_delete.set()
+ reset_thread.join(timeout=2)
+ start_thread.join(timeout=2)
+
+ assert results["reset"][1] == 200
+ assert results["start"][1] == 409
+ assert runner_called == []
+ assert project.graph_id is None
diff --git a/backend/tests/test_zep_graph_memory_updater.py b/backend/tests/test_zep_graph_memory_updater.py
new file mode 100644
index 00000000..5f5531b0
--- /dev/null
+++ b/backend/tests/test_zep_graph_memory_updater.py
@@ -0,0 +1,237 @@
+from types import SimpleNamespace
+import threading
+from queue import Queue
+
+import pytest
+
+from app.services import zep_graph_memory_updater as updater_module
+from app.services.zep_graph_memory_updater import (
+ AgentActivity,
+ ZepGraphMemoryManager,
+ ZepGraphMemoryUpdater,
+)
+
+
+def _activity(index=1, content="hello"):
+ return AgentActivity(
+ platform="twitter",
+ agent_id=index,
+ agent_name=f"Agent {index}",
+ action_type="CREATE_POST",
+ action_args={"content": content},
+ round_num=index,
+ timestamp="2026-07-22T12:00:00+08:00",
+ )
+
+
+def _client(add):
+ return SimpleNamespace(
+ graph=SimpleNamespace(
+ add=add,
+ episode=SimpleNamespace(
+ get=lambda **_kwargs: SimpleNamespace(processed=True)
+ ),
+ )
+ )
+
+
+def _updater(monkeypatch, add, simulation_id="sim-1"):
+ client = _client(add)
+ monkeypatch.setattr(updater_module, "get_zep_client", lambda _key: client)
+ updater = ZepGraphMemoryUpdater(
+ "graph-1",
+ api_key="test-key",
+ simulation_id=simulation_id,
+ )
+ updater.SEND_INTERVAL = 0
+ return updater
+
+
+def test_stop_drains_an_immediately_queued_tail_activity(monkeypatch):
+ writes = []
+ updater = _updater(
+ monkeypatch,
+ lambda **kwargs: writes.append(kwargs) or SimpleNamespace(uuid_="episode-1"),
+ )
+
+ updater.start()
+ updater.add_activity(_activity())
+ updater.stop()
+
+ assert len(writes) == 1
+ assert updater.get_stats()["items_sent"] == 1
+ assert updater.get_stats()["queue_size"] == 0
+
+
+def test_network_write_happens_outside_the_buffer_lock(monkeypatch):
+ lock_was_available = []
+ updater = None
+
+ def add(**_kwargs):
+ acquired = updater._buffer_lock.acquire(blocking=False)
+ lock_was_available.append(acquired)
+ if acquired:
+ updater._buffer_lock.release()
+ return SimpleNamespace(uuid_="episode-1")
+
+ updater = _updater(monkeypatch, add)
+ updater.start()
+ for index in range(updater.BATCH_SIZE):
+ updater.add_activity(_activity(index))
+ updater.stop()
+
+ assert lock_was_available == [True]
+
+
+def test_activity_episode_has_provenance_time_and_a_safe_size(monkeypatch):
+ writes = []
+ updater = _updater(
+ monkeypatch,
+ lambda **kwargs: writes.append(kwargs) or SimpleNamespace(uuid_="episode-1"),
+ simulation_id="sim-provenance",
+ )
+
+ updater._send_batch_activities(
+ [_activity(content="x" * 20_000)],
+ "twitter",
+ )
+
+ assert len(writes) == 1
+ write = writes[0]
+ assert len(write["data"]) <= updater.MAX_EPISODE_CHARS
+ assert write["created_at"] == "2026-07-22T12:00:00+08:00"
+ assert write["source_description"] == "MiroFish simulation activity batch"
+ assert write["metadata"]["simulation_id"] == "sim-provenance"
+ assert write["metadata"]["platform"] == "twitter"
+ assert write["metadata"]["activity_count"] == 1
+
+
+def test_failed_non_idempotent_write_is_reported_by_stop(monkeypatch):
+ def add(**_kwargs):
+ raise RuntimeError("write failed")
+
+ updater = _updater(monkeypatch, add)
+ updater.start()
+ updater.add_activity(_activity())
+
+ with pytest.raises(RuntimeError, match="ingestion is incomplete"):
+ updater.stop()
+
+ assert updater.get_stats()["failed_count"] == 1
+
+
+def test_failed_simulation_action_is_not_ingested(monkeypatch):
+ updater = _updater(
+ monkeypatch,
+ lambda **_kwargs: SimpleNamespace(uuid_="unused"),
+ )
+
+ updater.add_activity_from_dict(
+ {
+ "agent_id": 1,
+ "agent_name": "Agent",
+ "action_type": "CREATE_POST",
+ "action_args": {"content": "not actually posted"},
+ "success": False,
+ },
+ "twitter",
+ )
+
+ assert updater.get_stats()["queue_size"] == 0
+ assert updater.get_stats()["skipped_count"] == 1
+
+
+def test_stop_cannot_finish_between_acceptance_check_and_enqueue(monkeypatch):
+ writes = []
+ updater = _updater(
+ monkeypatch,
+ lambda **kwargs: writes.append(kwargs) or SimpleNamespace(uuid_="episode-1"),
+ )
+
+ put_entered = threading.Event()
+ allow_put = threading.Event()
+
+ class BlockingQueue(Queue):
+ def put(self, item, block=True, timeout=None):
+ put_entered.set()
+ assert allow_put.wait(timeout=2)
+ return super().put(item, block=block, timeout=timeout)
+
+ updater._activity_queue = BlockingQueue()
+ updater.start()
+ producer = threading.Thread(target=updater.add_activity, args=(_activity(),))
+ producer.start()
+ assert put_entered.wait(timeout=1)
+
+ stopper = threading.Thread(target=updater.stop)
+ stopper.start()
+ stopper.join(timeout=0.1)
+ assert stopper.is_alive()
+
+ allow_put.set()
+ producer.join(timeout=2)
+ stopper.join(timeout=2)
+
+ assert not producer.is_alive()
+ assert not stopper.is_alive()
+ assert len(writes) == 1
+
+
+def test_pending_episode_wait_has_a_deadline(monkeypatch):
+ updater = _updater(
+ monkeypatch,
+ lambda **_kwargs: SimpleNamespace(uuid_="episode-1"),
+ )
+ updater._pending_episode_uuids = ["episode-1"]
+ updater.client.graph.episode.get = lambda **_kwargs: SimpleNamespace(
+ processed=False
+ )
+ timestamps = iter([0.0, 2.0])
+ monkeypatch.setattr(updater_module.Config, "ZEP_INGESTION_TIMEOUT_SECONDS", 1)
+ monkeypatch.setattr(updater_module.time, "time", lambda: next(timestamps))
+ monkeypatch.setattr(updater_module.time, "sleep", lambda _seconds: None)
+
+ with pytest.raises(TimeoutError, match="pending"):
+ updater._wait_for_pending_episodes()
+
+
+def test_explicit_graph_destruction_can_discard_a_stopped_failed_updater():
+ updater = SimpleNamespace(
+ graph_id="graph-1",
+ _running=False,
+ _worker_thread=SimpleNamespace(is_alive=lambda: False),
+ )
+ ZepGraphMemoryManager._updaters["sim-failed"] = updater
+ try:
+ assert ZepGraphMemoryManager.discard_inactive_updater("sim-failed") is True
+ assert "sim-failed" not in ZepGraphMemoryManager._updaters
+ finally:
+ ZepGraphMemoryManager._updaters.pop("sim-failed", None)
+
+
+def test_flush_deadline_keeps_unattempted_platform_for_a_safe_retry(monkeypatch):
+ now = [0.0]
+ writes = []
+
+ def add(**kwargs):
+ writes.append(kwargs)
+ now[0] = 2.0
+ return SimpleNamespace(uuid_=f"episode-{len(writes)}")
+
+ updater = _updater(monkeypatch, add)
+ updater._platform_buffers["twitter"] = [_activity(1)]
+ reddit_activity = _activity(2)
+ reddit_activity.platform = "reddit"
+ updater._platform_buffers["reddit"] = [reddit_activity]
+ monkeypatch.setattr(updater_module.time, "time", lambda: now[0])
+
+ with pytest.raises(TimeoutError, match="deadline"):
+ updater._flush_remaining(deadline=1.0)
+
+ assert updater._platform_buffers["twitter"] == []
+ assert updater._platform_buffers["reddit"] == [reddit_activity]
+
+ now[0] = 0.0
+ updater._flush_remaining(deadline=1.0)
+ assert updater._platform_buffers["reddit"] == []
+ assert len(writes) == 2
diff --git a/backend/tests/test_zep_report_barrier.py b/backend/tests/test_zep_report_barrier.py
new file mode 100644
index 00000000..cda337dd
--- /dev/null
+++ b/backend/tests/test_zep_report_barrier.py
@@ -0,0 +1,297 @@
+from types import SimpleNamespace
+
+import pytest
+from flask import Flask
+
+from app.api import graph as graph_api
+from app.api import report as report_api
+from app.api import simulation as simulation_api
+from app.models.project import ProjectStatus
+from app.services.simulation_manager import SimulationStatus
+from app.services.simulation_runner import RunnerStatus
+from app.utils.zep_lifecycle import (
+ get_graph_readers,
+ unregister_graph_reader,
+)
+
+
+def _json_result(result):
+ if isinstance(result, tuple):
+ response, status = result
+ else:
+ response, status = result, result.status_code
+ return response.get_json(), status
+
+
+def test_report_generation_waits_for_zep_ingestion(monkeypatch):
+ simulation = SimpleNamespace(project_id="proj-1", graph_id="graph-1")
+ monkeypatch.setattr(
+ report_api,
+ "SimulationManager",
+ lambda: SimpleNamespace(
+ get_simulation=lambda _simulation_id: simulation
+ ),
+ )
+ monkeypatch.setattr(
+ report_api.ReportManager,
+ "get_report_by_simulation",
+ classmethod(lambda _cls, _simulation_id: None),
+ )
+ monkeypatch.setattr(
+ report_api.SimulationRunner,
+ "get_run_state",
+ classmethod(
+ lambda _cls, _simulation_id: SimpleNamespace(
+ runner_status=RunnerStatus.STOPPING
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ report_api.ZepGraphMemoryManager,
+ "get_updater",
+ classmethod(lambda _cls, _simulation_id: object()),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context(
+ "/api/report/generate",
+ method="POST",
+ json={"simulation_id": "sim-1"},
+ ):
+ body, status = _json_result(report_api.generate_report())
+
+ assert status == 409
+ assert body["ingestion_pending"] is True
+
+
+def test_active_rerun_does_not_return_a_stale_completed_report(monkeypatch):
+ simulation = SimpleNamespace(project_id="proj-1", graph_id="graph-1")
+ monkeypatch.setattr(
+ report_api,
+ "SimulationManager",
+ lambda: SimpleNamespace(
+ get_simulation=lambda _simulation_id: simulation
+ ),
+ )
+ monkeypatch.setattr(
+ report_api.ReportManager,
+ "get_report_by_simulation",
+ classmethod(
+ lambda _cls, _simulation_id: SimpleNamespace(
+ report_id="old-report",
+ status=report_api.ReportStatus.COMPLETED,
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ report_api.SimulationRunner,
+ "get_run_state",
+ classmethod(
+ lambda _cls, _simulation_id: SimpleNamespace(
+ runner_status=RunnerStatus.STOPPING
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ report_api.ZepGraphMemoryManager,
+ "get_updater",
+ classmethod(lambda _cls, _simulation_id: object()),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context(
+ "/api/report/generate",
+ method="POST",
+ json={"simulation_id": "sim-1"},
+ ):
+ body, status = _json_result(report_api.generate_report())
+
+ assert status == 409
+ assert body["ingestion_pending"] is True
+
+
+def test_failed_ingestion_cannot_generate_a_report_after_restart(monkeypatch):
+ simulation = SimpleNamespace(project_id="proj-1", graph_id="graph-1")
+ monkeypatch.setattr(
+ report_api,
+ "SimulationManager",
+ lambda: SimpleNamespace(
+ get_simulation=lambda _simulation_id: simulation
+ ),
+ )
+ monkeypatch.setattr(
+ report_api.SimulationRunner,
+ "get_run_state",
+ classmethod(
+ lambda _cls, _simulation_id: SimpleNamespace(
+ runner_status=RunnerStatus.FAILED
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ report_api.ZepGraphMemoryManager,
+ "get_updater",
+ classmethod(lambda _cls, _simulation_id: None),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context(
+ "/api/report/generate",
+ method="POST",
+ json={"simulation_id": "sim-1"},
+ ):
+ body, status = _json_result(report_api.generate_report())
+
+ assert status == 409
+ assert "successfully completed" in body["error"]
+
+
+def test_report_reader_lease_blocks_graph_start_and_delete(monkeypatch):
+ simulation = SimpleNamespace(
+ simulation_id="sim-1",
+ project_id="proj-1",
+ graph_id="graph-1",
+ status=SimulationStatus.READY,
+ )
+ project = SimpleNamespace(
+ project_id="proj-1",
+ graph_id="graph-1",
+ status=ProjectStatus.GRAPH_COMPLETED,
+ simulation_requirement="mock requirement",
+ )
+ run_state = SimpleNamespace(runner_status=RunnerStatus.COMPLETED)
+ worker_targets = []
+ runner_calls = []
+
+ class Tasks:
+ def create_task(self, **_kwargs):
+ return "task-1"
+
+ def update_task(self, *_args, **_kwargs):
+ pass
+
+ def complete_task(self, *_args, **_kwargs):
+ pass
+
+ def fail_task(self, *_args, **_kwargs):
+ pass
+
+ class ParkedThread:
+ def __init__(self, *, target, daemon):
+ assert daemon is True
+ self.target = target
+
+ def start(self):
+ worker_targets.append(self.target)
+
+ class Agent:
+ def __init__(self, **_kwargs):
+ pass
+
+ def generate_report(self, *, progress_callback, report_id):
+ progress_callback("mock", 100, "done")
+ return SimpleNamespace(
+ report_id=report_id,
+ status=report_api.ReportStatus.COMPLETED,
+ error=None,
+ )
+
+ monkeypatch.setattr(
+ report_api,
+ "SimulationManager",
+ lambda: SimpleNamespace(
+ get_simulation=lambda _simulation_id: simulation
+ ),
+ )
+ monkeypatch.setattr(
+ simulation_api,
+ "SimulationManager",
+ lambda: SimpleNamespace(
+ get_simulation=lambda _simulation_id: simulation
+ ),
+ )
+ monkeypatch.setattr(
+ report_api.ProjectManager,
+ "get_project",
+ classmethod(lambda _cls, _project_id: project),
+ )
+ monkeypatch.setattr(
+ report_api.SimulationRunner,
+ "get_run_state",
+ classmethod(lambda _cls, _simulation_id: run_state),
+ )
+ monkeypatch.setattr(
+ report_api.ZepGraphMemoryManager,
+ "get_updater",
+ classmethod(lambda _cls, _simulation_id: None),
+ )
+ monkeypatch.setattr(
+ report_api.ReportManager,
+ "get_report_by_simulation",
+ classmethod(lambda _cls, _simulation_id: None),
+ )
+ monkeypatch.setattr(
+ report_api.ReportManager,
+ "save_report",
+ classmethod(lambda _cls, _report: None),
+ )
+ monkeypatch.setattr(report_api, "TaskManager", Tasks)
+ monkeypatch.setattr(report_api, "ReportAgent", Agent)
+ monkeypatch.setattr(report_api.threading, "Thread", ParkedThread)
+ monkeypatch.setattr(
+ simulation_api.SimulationRunner,
+ "start_simulation",
+ classmethod(
+ lambda _cls, **_kwargs: runner_calls.append(True)
+ ),
+ )
+ monkeypatch.setattr(
+ graph_api.ZepGraphMemoryManager,
+ "get_simulation_ids_for_graph",
+ classmethod(lambda _cls, _graph_id: []),
+ )
+ monkeypatch.setattr(
+ graph_api,
+ "SimulationManager",
+ lambda: SimpleNamespace(list_simulations=lambda: []),
+ )
+
+ app = Flask(__name__)
+ report_id = None
+ try:
+ with app.test_request_context(
+ "/api/report/generate",
+ method="POST",
+ json={"simulation_id": "sim-1"},
+ ):
+ body, status = _json_result(report_api.generate_report())
+ assert status == 200
+ report_id = body["data"]["report_id"]
+ assert get_graph_readers("graph-1") == [report_id]
+ assert len(worker_targets) == 1
+
+ with app.test_request_context(
+ "/api/simulation/start",
+ method="POST",
+ json={
+ "simulation_id": "sim-1",
+ "enable_graph_memory_update": True,
+ },
+ ):
+ start_body, start_status = _json_result(
+ simulation_api.start_simulation()
+ )
+ assert start_status == 409
+ assert start_body["active_reports"] == [report_id]
+ assert runner_calls == []
+
+ with pytest.raises(graph_api.GraphInUseError, match=f"report:{report_id}"):
+ graph_api._delete_cloud_graph_if_present("graph-1")
+
+ # Let the parked background report finish; its finally block must
+ # release the lease even if report generation fails.
+ worker_targets[0]()
+ assert get_graph_readers("graph-1") == []
+ finally:
+ if report_id:
+ unregister_graph_reader("graph-1", report_id)
diff --git a/backend/tests/test_zep_retry_and_client.py b/backend/tests/test_zep_retry_and_client.py
new file mode 100644
index 00000000..5e8d4494
--- /dev/null
+++ b/backend/tests/test_zep_retry_and_client.py
@@ -0,0 +1,102 @@
+from types import SimpleNamespace
+
+import pytest
+from zep_cloud.core.api_error import ApiError as ZepApiError
+
+from app.utils import zep
+from app import config
+
+
+def test_permanent_zep_errors_fail_without_retry():
+ calls = []
+
+ def operation():
+ calls.append(True)
+ raise ZepApiError(status_code=400, body={"message": "bad query"})
+
+ with pytest.raises(ZepApiError):
+ zep.call_zep_read_with_retry(
+ operation,
+ operation_name="permanent failure",
+ sleep=lambda _seconds: None,
+ )
+
+ assert len(calls) == 1
+
+
+def test_rate_limit_retry_respects_retry_after():
+ calls = []
+ sleeps = []
+
+ def operation():
+ calls.append(True)
+ if len(calls) == 1:
+ raise ZepApiError(
+ status_code=429,
+ headers={"Retry-After": "7"},
+ body={"message": "slow down"},
+ )
+ return "ok"
+
+ result = zep.call_zep_read_with_retry(
+ operation,
+ operation_name="rate limited read",
+ sleep=sleeps.append,
+ )
+
+ assert result == "ok"
+ assert len(calls) == 2
+ assert sleeps == [7.0]
+
+
+def test_zep_client_is_shared_and_uses_an_explicit_timeout(monkeypatch):
+ created = []
+
+ def fake_zep(**kwargs):
+ created.append(kwargs)
+ return SimpleNamespace(kwargs=kwargs)
+
+ monkeypatch.delenv("ZEP_API_URL", raising=False)
+ monkeypatch.setattr(zep, "Zep", fake_zep)
+ zep.clear_zep_client_cache()
+
+ first = zep.get_zep_client(" test-key ", timeout=12)
+ second = zep.get_zep_client("test-key", timeout=12)
+
+ assert first is second
+ assert created == [{
+ "api_key": "test-key",
+ "base_url": zep.ZEP_CLOUD_BASE_URL,
+ "timeout": 12.0,
+ }]
+ zep.clear_zep_client_cache()
+
+
+def test_zep_client_rejects_self_hosted_endpoint_override(monkeypatch):
+ monkeypatch.setenv("ZEP_API_URL", "https://example.invalid")
+
+ with pytest.raises(ValueError, match="ZEP_API_URL"):
+ zep.get_zep_client("test-key")
+
+
+def test_malformed_timeout_config_is_reported_without_import_failure():
+ value, error = config._parse_number(
+ "ZEP_REQUEST_TIMEOUT_SECONDS",
+ "not-a-number",
+ default=30.0,
+ cast=float,
+ )
+
+ assert value == 30.0
+ assert "must be a number" in error
+
+
+def test_zep_client_rejects_a_recorded_timeout_parse_error(monkeypatch):
+ monkeypatch.setattr(
+ zep.Config,
+ "_ZEP_CONFIG_PARSE_ERRORS",
+ ("ZEP_REQUEST_TIMEOUT_SECONDS must be a number",),
+ )
+
+ with pytest.raises(ValueError, match="must be a number"):
+ zep.get_zep_client("test-key")
diff --git a/backend/tests/test_zep_simulation_barrier.py b/backend/tests/test_zep_simulation_barrier.py
new file mode 100644
index 00000000..63289fce
--- /dev/null
+++ b/backend/tests/test_zep_simulation_barrier.py
@@ -0,0 +1,484 @@
+from types import SimpleNamespace
+import json
+
+import pytest
+from flask import Flask
+
+from app.api import simulation as simulation_api
+from app.services import simulation_runner as runner_module
+from app.services.simulation_manager import SimulationStatus
+from app.services.simulation_runner import (
+ RunnerStatus,
+ SimulationRunState,
+ SimulationRunner,
+ SimulationStopPending,
+)
+
+
+def test_manual_stop_surfaces_graph_ingestion_failure(monkeypatch):
+ state = SimulationRunState(
+ simulation_id="sim-1",
+ runner_status=RunnerStatus.RUNNING,
+ )
+ saved = []
+ monkeypatch.setattr(
+ SimulationRunner,
+ "get_run_state",
+ classmethod(lambda _cls, _simulation_id: state),
+ )
+ monkeypatch.setattr(
+ SimulationRunner,
+ "_save_run_state",
+ classmethod(lambda _cls, value: saved.append(value.runner_status)),
+ )
+ monkeypatch.setattr(
+ runner_module.ZepGraphMemoryManager,
+ "stop_updater",
+ classmethod(
+ lambda _cls, _simulation_id: (_ for _ in ()).throw(
+ RuntimeError("ingestion incomplete")
+ )
+ ),
+ )
+ SimulationRunner._processes.pop("sim-1", None)
+ SimulationRunner._graph_memory_enabled["sim-1"] = True
+
+ try:
+ with pytest.raises(RuntimeError, match="ingestion incomplete"):
+ SimulationRunner.stop_simulation("sim-1")
+
+ assert state.runner_status == RunnerStatus.FAILED
+ assert "ingestion incomplete" in state.error
+ assert saved[-1] == RunnerStatus.FAILED
+ finally:
+ SimulationRunner._graph_memory_enabled.pop("sim-1", None)
+ SimulationRunner._manual_stop_requests.discard("sim-1")
+
+
+def test_platform_completion_does_not_publish_terminal_success_before_barrier(
+ monkeypatch, tmp_path
+):
+ simulation_id = "sim-1"
+ sim_dir = tmp_path / simulation_id / "twitter"
+ sim_dir.mkdir(parents=True)
+ log_path = sim_dir / "actions.jsonl"
+ log_path.write_text(
+ '{"event_type":"simulation_end","total_rounds":1,"total_actions":0}\n',
+ encoding="utf-8",
+ )
+ monkeypatch.setattr(SimulationRunner, "RUN_STATE_DIR", str(tmp_path))
+ state = SimulationRunState(
+ simulation_id=simulation_id,
+ runner_status=RunnerStatus.RUNNING,
+ twitter_running=True,
+ )
+
+ SimulationRunner._read_action_log(str(log_path), 0, state, "twitter")
+
+ assert state.twitter_completed is True
+ assert state.runner_status == RunnerStatus.RUNNING
+
+
+def test_manual_stop_timeout_leaves_monitor_owned_state_stopping(monkeypatch):
+ state = SimulationRunState(
+ simulation_id="sim-timeout",
+ runner_status=RunnerStatus.RUNNING,
+ )
+
+ class Monitor:
+ def join(self, timeout):
+ assert timeout >= 30
+
+ def is_alive(self):
+ return True
+
+ monkeypatch.setattr(
+ SimulationRunner,
+ "get_run_state",
+ classmethod(lambda _cls, _simulation_id: state),
+ )
+ monkeypatch.setattr(
+ SimulationRunner,
+ "_save_run_state",
+ classmethod(lambda _cls, _state: None),
+ )
+ SimulationRunner._monitor_threads["sim-timeout"] = Monitor()
+ SimulationRunner._processes.pop("sim-timeout", None)
+ SimulationRunner._graph_memory_enabled.pop("sim-timeout", None)
+
+ try:
+ with pytest.raises(TimeoutError, match="仍在停止中"):
+ SimulationRunner.stop_simulation("sim-timeout")
+ assert state.runner_status == RunnerStatus.STOPPING
+ finally:
+ SimulationRunner._monitor_threads.pop("sim-timeout", None)
+ SimulationRunner._manual_stop_requests.discard("sim-timeout")
+
+
+def test_failed_ingestion_finalization_can_be_retried(monkeypatch):
+ state = SimulationRunState(
+ simulation_id="sim-retry",
+ runner_status=RunnerStatus.FAILED,
+ error="first drain timed out",
+ )
+ monkeypatch.setattr(
+ SimulationRunner,
+ "get_run_state",
+ classmethod(lambda _cls, _simulation_id: state),
+ )
+ monkeypatch.setattr(
+ SimulationRunner,
+ "_save_run_state",
+ classmethod(lambda _cls, _state: None),
+ )
+ monkeypatch.setattr(
+ runner_module.ZepGraphMemoryManager,
+ "get_updater",
+ classmethod(lambda _cls, _simulation_id: object()),
+ )
+ monkeypatch.setattr(
+ runner_module.ZepGraphMemoryManager,
+ "stop_updater",
+ classmethod(lambda _cls, _simulation_id: None),
+ )
+ SimulationRunner._graph_memory_enabled["sim-retry"] = True
+ SimulationRunner._monitor_threads.pop("sim-retry", None)
+
+ try:
+ result = SimulationRunner.stop_simulation("sim-retry")
+ assert result.runner_status == RunnerStatus.STOPPED
+ assert result.error is None
+ finally:
+ SimulationRunner._graph_memory_enabled.pop("sim-retry", None)
+ SimulationRunner._manual_stop_requests.discard("sim-retry")
+
+
+def test_stop_api_keeps_pending_finalization_out_of_failed_state(monkeypatch):
+ simulation = SimpleNamespace(status=SimulationStatus.STOPPING, error=None)
+ saved = []
+ monkeypatch.setattr(
+ simulation_api.SimulationRunner,
+ "stop_simulation",
+ classmethod(
+ lambda _cls, _simulation_id: (_ for _ in ()).throw(
+ SimulationStopPending("still draining")
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ simulation_api,
+ "SimulationManager",
+ lambda: SimpleNamespace(
+ get_simulation=lambda _simulation_id: simulation,
+ _save_simulation_state=lambda state: saved.append(state.status),
+ ),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context(
+ "/api/simulation/stop",
+ method="POST",
+ json={"simulation_id": "sim-pending"},
+ ):
+ response, status = simulation_api.stop_simulation()
+
+ assert status == 202
+ assert response.get_json()["pending"] is True
+ assert simulation.status == SimulationStatus.STOPPING
+ assert saved == []
+
+
+ "field",
+ ["force", "enable_graph_memory_update"],
+)
+def test_simulation_start_rejects_string_booleans(field):
+ app = Flask(__name__)
+ with app.test_request_context(
+ "/api/simulation/start",
+ method="POST",
+ json={"simulation_id": "sim-1", field: "false"},
+ ):
+ response, status = simulation_api.start_simulation()
+
+ assert status == 400
+ assert "JSON boolean" in response.get_json()["error"]
+
+
+def test_force_restart_does_not_continue_while_old_ingestion_is_pending(monkeypatch):
+ simulation = SimpleNamespace(
+ simulation_id="sim-1",
+ project_id="proj-1",
+ graph_id="graph-1",
+ status=SimulationStatus.STOPPING,
+ )
+ cleanup_called = []
+ monkeypatch.setattr(
+ simulation_api,
+ "SimulationManager",
+ lambda: SimpleNamespace(
+ get_simulation=lambda _simulation_id: simulation,
+ _save_simulation_state=lambda _state: None,
+ ),
+ )
+ monkeypatch.setattr(
+ simulation_api,
+ "_check_simulation_prepared",
+ lambda _simulation_id: (True, {}),
+ )
+ monkeypatch.setattr(
+ simulation_api.SimulationRunner,
+ "get_run_state",
+ classmethod(
+ lambda _cls, _simulation_id: SimpleNamespace(
+ runner_status=RunnerStatus.STOPPING
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ simulation_api.SimulationRunner,
+ "stop_simulation",
+ classmethod(
+ lambda _cls, _simulation_id: (_ for _ in ()).throw(
+ SimulationStopPending("still draining")
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ simulation_api.SimulationRunner,
+ "cleanup_simulation_logs",
+ classmethod(
+ lambda _cls, _simulation_id: cleanup_called.append(True)
+ ),
+ )
+ monkeypatch.setattr(
+ simulation_api.ZepGraphMemoryManager,
+ "get_updater",
+ classmethod(lambda _cls, _simulation_id: object()),
+ )
+
+ app = Flask(__name__)
+ with app.test_request_context(
+ "/api/simulation/start",
+ method="POST",
+ json={"simulation_id": "sim-1", "force": True},
+ ):
+ response, status = simulation_api.start_simulation()
+
+ assert status == 409
+ assert response.get_json()["pending"] is True
+ assert cleanup_called == []
+
+
+def test_monitor_start_failure_terminates_the_spawned_process(monkeypatch, tmp_path):
+ simulation_id = "sim-start-failure"
+ sim_dir = tmp_path / "runs" / simulation_id
+ scripts_dir = tmp_path / "scripts"
+ sim_dir.mkdir(parents=True)
+ scripts_dir.mkdir()
+ (sim_dir / "simulation_config.json").write_text(
+ json.dumps({
+ "time_config": {
+ "total_simulation_hours": 1,
+ "minutes_per_round": 60,
+ }
+ }),
+ encoding="utf-8",
+ )
+ (scripts_dir / "run_twitter_simulation.py").write_text("pass\n", encoding="utf-8")
+
+ class Process:
+ pid = 123
+
+ def poll(self):
+ return None
+
+ class BrokenThread:
+ def __init__(self, **_kwargs):
+ pass
+
+ def start(self):
+ raise RuntimeError("monitor failed")
+
+ terminated = []
+ monkeypatch.setattr(SimulationRunner, "RUN_STATE_DIR", str(tmp_path / "runs"))
+ monkeypatch.setattr(SimulationRunner, "SCRIPTS_DIR", str(scripts_dir))
+ monkeypatch.setattr(runner_module.subprocess, "Popen", lambda *_args, **_kwargs: Process())
+ monkeypatch.setattr(runner_module.threading, "Thread", BrokenThread)
+ monkeypatch.setattr(
+ SimulationRunner,
+ "_terminate_process",
+ classmethod(lambda _cls, _process, sim_id: terminated.append(sim_id)),
+ )
+ monkeypatch.setattr(
+ SimulationRunner,
+ "_sync_simulation_status",
+ classmethod(lambda _cls, *_args, **_kwargs: None),
+ )
+
+ try:
+ with pytest.raises(RuntimeError, match="monitor failed"):
+ SimulationRunner.start_simulation(
+ simulation_id,
+ platform="twitter",
+ enable_graph_memory_update=False,
+ )
+ assert terminated == [simulation_id]
+ assert simulation_id not in SimulationRunner._processes
+ assert simulation_id not in SimulationRunner._action_queues
+ assert simulation_id not in SimulationRunner._stdout_files
+ finally:
+ SimulationRunner._run_states.pop(simulation_id, None)
+ SimulationRunner._processes.pop(simulation_id, None)
+ SimulationRunner._action_queues.pop(simulation_id, None)
+ SimulationRunner._stdout_files.pop(simulation_id, None)
+ SimulationRunner._stderr_files.pop(simulation_id, None)
+ SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
+
+
+def test_shutdown_terminates_producer_before_tail_read_and_updater_drain(
+ monkeypatch,
+):
+ simulation_id = "sim-shutdown-order"
+ state = SimulationRunState(
+ simulation_id=simulation_id,
+ runner_status=RunnerStatus.RUNNING,
+ )
+ events = []
+
+ class Process:
+ pid = 123
+ stopped = False
+
+ def poll(self):
+ return 0 if self.stopped else None
+
+ process = Process()
+
+ class Monitor:
+ alive = True
+
+ def join(self, timeout):
+ assert timeout >= 30
+ events.extend(["tail-read", "updater-drain"])
+ state.runner_status = RunnerStatus.STOPPED
+ SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
+ self.alive = False
+
+ def is_alive(self):
+ return self.alive
+
+ monkeypatch.setattr(
+ SimulationRunner,
+ "get_run_state",
+ classmethod(lambda _cls, _simulation_id: state),
+ )
+ monkeypatch.setattr(
+ SimulationRunner,
+ "_save_run_state",
+ classmethod(lambda _cls, _state: None),
+ )
+ monkeypatch.setattr(
+ SimulationRunner,
+ "_sync_simulation_status",
+ classmethod(lambda _cls, *_args, **_kwargs: None),
+ )
+ monkeypatch.setattr(
+ SimulationRunner,
+ "_terminate_process",
+ classmethod(
+ lambda _cls, proc, _simulation_id, **_kwargs: (
+ events.append("producer-terminate"),
+ setattr(proc, "stopped", True),
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ runner_module.ZepGraphMemoryManager,
+ "get_simulation_ids",
+ classmethod(lambda _cls: [simulation_id]),
+ )
+ updater = object()
+ monkeypatch.setattr(
+ runner_module.ZepGraphMemoryManager,
+ "get_updater",
+ classmethod(lambda _cls, _simulation_id: updater),
+ )
+
+ SimulationRunner._cleanup_done = False
+ SimulationRunner._processes[simulation_id] = process
+ SimulationRunner._monitor_threads[simulation_id] = Monitor()
+ SimulationRunner._graph_memory_enabled[simulation_id] = True
+ try:
+ SimulationRunner.cleanup_all_simulations()
+ assert events == [
+ "producer-terminate",
+ "tail-read",
+ "updater-drain",
+ ]
+ assert state.runner_status == RunnerStatus.STOPPED
+ finally:
+ SimulationRunner._cleanup_done = False
+ SimulationRunner._processes.pop(simulation_id, None)
+ SimulationRunner._monitor_threads.pop(simulation_id, None)
+ SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
+ SimulationRunner._manual_stop_requests.discard(simulation_id)
+
+
+def test_shutdown_drain_failure_remains_failed_and_retryable(monkeypatch):
+ simulation_id = "sim-shutdown-failure"
+ state = SimulationRunState(
+ simulation_id=simulation_id,
+ runner_status=RunnerStatus.RUNNING,
+ )
+ updater = object()
+
+ monkeypatch.setattr(
+ SimulationRunner,
+ "get_run_state",
+ classmethod(lambda _cls, _simulation_id: state),
+ )
+ monkeypatch.setattr(
+ SimulationRunner,
+ "_save_run_state",
+ classmethod(lambda _cls, _state: None),
+ )
+ monkeypatch.setattr(
+ SimulationRunner,
+ "_sync_simulation_status",
+ classmethod(lambda _cls, *_args, **_kwargs: None),
+ )
+ monkeypatch.setattr(
+ runner_module.ZepGraphMemoryManager,
+ "get_simulation_ids",
+ classmethod(lambda _cls: [simulation_id]),
+ )
+ monkeypatch.setattr(
+ runner_module.ZepGraphMemoryManager,
+ "get_updater",
+ classmethod(lambda _cls, _simulation_id: updater),
+ )
+ monkeypatch.setattr(
+ runner_module.ZepGraphMemoryManager,
+ "stop_updater",
+ classmethod(
+ lambda _cls, _simulation_id: (_ for _ in ()).throw(
+ RuntimeError("drain incomplete")
+ )
+ ),
+ )
+
+ SimulationRunner._cleanup_done = False
+ SimulationRunner._graph_memory_enabled[simulation_id] = True
+ SimulationRunner._monitor_threads.pop(simulation_id, None)
+ SimulationRunner._processes.pop(simulation_id, None)
+ try:
+ SimulationRunner.cleanup_all_simulations()
+ assert state.runner_status == RunnerStatus.FAILED
+ assert "drain incomplete" in state.error
+ assert SimulationRunner._graph_memory_enabled[simulation_id] is True
+ assert SimulationRunner._cleanup_done is False
+ finally:
+ SimulationRunner._cleanup_done = False
+ SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
+ SimulationRunner._manual_stop_requests.discard(simulation_id)
diff --git a/backend/uv.lock b/backend/uv.lock
index b2f853cc..7f606a13 100644
--- a/backend/uv.lock
+++ b/backend/uv.lock
@@ -994,6 +994,7 @@ dependencies = [
{ name = "charset-normalizer" },
{ name = "flask" },
{ name = "flask-cors" },
+ { name = "httpx" },
{ name = "openai" },
{ name = "pydantic" },
{ name = "pymupdf" },
@@ -1022,6 +1023,7 @@ requires-dist = [
{ name = "charset-normalizer", specifier = ">=3.0.0" },
{ name = "flask", specifier = ">=3.0.0" },
{ name = "flask-cors", specifier = ">=6.0.0" },
+ { name = "httpx", specifier = ">=0.27.0" },
{ name = "openai", specifier = ">=1.0.0" },
{ name = "pipreqs", marker = "extra == 'dev'", specifier = ">=0.5.0" },
{ name = "pydantic", specifier = ">=2.0.0" },
@@ -1029,7 +1031,7 @@ requires-dist = [
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" },
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" },
{ name = "python-dotenv", specifier = ">=1.0.0" },
- { name = "zep-cloud", specifier = "==3.13.0" },
+ { name = "zep-cloud", specifier = "==3.25.0" },
]
provides-extras = ["dev"]
@@ -2775,7 +2777,7 @@ wheels = [
[[package]]
name = "zep-cloud"
-version = "3.13.0"
+version = "3.25.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "httpx" },
@@ -2784,7 +2786,7 @@ dependencies = [
{ name = "python-dateutil" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/32/c7/c835debf13302f8aaf8d0561ac6ff5a9bc15cc140cd692a1330fb1900c55/zep_cloud-3.13.0.tar.gz", hash = "sha256:c55d9c511773bb2177ae8e08546141404f87d2099affafabd7ec4b4505763e48", size = 63116, upload-time = "2025-11-20T15:25:40.745Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/81/0d/7a866e1e1e5f0f1353eedf3d0e17df9b3cf362c494bccd41eb121b49c43f/zep_cloud-3.25.0.tar.gz", hash = "sha256:a77867e02c2a9036a20623e85d03415e237e76510b6e05ff2e9ab2cdced0aeb9", size = 94772, upload-time = "2026-07-16T00:46:43.918Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/f7/e1/bbf03c6c8007c0cb238780e7fc6d8e1a52633893933a41aa09678618985a/zep_cloud-3.13.0-py3-none-any.whl", hash = "sha256:b2fbdeef73e262194c8f67b58f76471de6ee87e1a629541a09d8f7bbf475f12b", size = 110601, upload-time = "2025-11-20T15:25:38.484Z" },
+ { url = "https://files.pythonhosted.org/packages/77/fc/5ca2bdd20b83bd62bc2708580f756e704f69704709e0ed43e735aa590c1e/zep_cloud-3.25.0-py3-none-any.whl", hash = "sha256:94d9599038b154af9ad33cab4b4873ed9adb1bcaa48fc4c41cbd407c1a657ef5", size = 166438, upload-time = "2026-07-16T00:46:42.39Z" },
]
diff --git a/frontend/src/api/graph.js b/frontend/src/api/graph.js
index ef90a2b6..85669559 100644
--- a/frontend/src/api/graph.js
+++ b/frontend/src/api/graph.js
@@ -1,4 +1,4 @@
-import service, { requestWithRetry } from './index'
+import service from './index'
/**
* 生成本体(上传文档和模拟需求)
@@ -6,16 +6,14 @@ import service, { requestWithRetry } from './index'
* @returns {Promise}
*/
export function generateOntology(formData) {
- return requestWithRetry(() =>
- service({
- url: '/api/graph/ontology/generate',
- method: 'post',
- data: formData,
- headers: {
- 'Content-Type': 'multipart/form-data'
- }
- })
- )
+ return service({
+ url: '/api/graph/ontology/generate',
+ method: 'post',
+ data: formData,
+ headers: {
+ 'Content-Type': 'multipart/form-data'
+ }
+ })
}
/**
@@ -24,13 +22,11 @@ export function generateOntology(formData) {
* @returns {Promise}
*/
export function buildGraph(data) {
- return requestWithRetry(() =>
- service({
- url: '/api/graph/build',
- method: 'post',
- data
- })
- )
+ return service({
+ url: '/api/graph/build',
+ method: 'post',
+ data
+ })
}
/**
diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js
index e840e116..807a0b4e 100644
--- a/frontend/src/api/index.js
+++ b/frontend/src/api/index.js
@@ -52,18 +52,4 @@ service.interceptors.response.use(
}
)
-// 带重试的请求函数
-export const requestWithRetry = async (requestFn, maxRetries = 3, delay = 1000) => {
- for (let i = 0; i < maxRetries; i++) {
- try {
- return await requestFn()
- } catch (error) {
- if (i === maxRetries - 1) throw error
-
- console.warn(`Request failed, retrying (${i + 1}/${maxRetries})...`)
- await new Promise(resolve => setTimeout(resolve, delay * Math.pow(2, i)))
- }
- }
-}
-
export default service
diff --git a/frontend/src/api/report.js b/frontend/src/api/report.js
index c89a67d8..9b61ce72 100644
--- a/frontend/src/api/report.js
+++ b/frontend/src/api/report.js
@@ -1,11 +1,11 @@
-import service, { requestWithRetry } from './index'
+import service from './index'
/**
* 开始报告生成
* @param {Object} data - { simulation_id, force_regenerate? }
*/
export const generateReport = (data) => {
- return requestWithRetry(() => service.post('/api/report/generate', data), 3, 1000)
+ return service.post('/api/report/generate', data)
}
/**
@@ -47,5 +47,5 @@ export const getReport = (reportId) => {
* @param {Object} data - { simulation_id, message, chat_history? }
*/
export const chatWithReport = (data) => {
- return requestWithRetry(() => service.post('/api/report/chat', data), 3, 1000)
+ return service.post('/api/report/chat', data)
}
diff --git a/frontend/src/api/simulation.js b/frontend/src/api/simulation.js
index 4dddd073..d495afae 100644
--- a/frontend/src/api/simulation.js
+++ b/frontend/src/api/simulation.js
@@ -1,11 +1,11 @@
-import service, { requestWithRetry } from './index'
+import service from './index'
/**
* 创建模拟
* @param {Object} data - { project_id, graph_id?, enable_twitter?, enable_reddit? }
*/
export const createSimulation = (data) => {
- return requestWithRetry(() => service.post('/api/simulation/create', data), 3, 1000)
+ return service.post('/api/simulation/create', data)
}
/**
@@ -13,7 +13,7 @@ export const createSimulation = (data) => {
* @param {Object} data - { simulation_id, entity_types?, use_llm_for_profiles?, parallel_profile_count?, force_regenerate? }
*/
export const prepareSimulation = (data) => {
- return requestWithRetry(() => service.post('/api/simulation/prepare', data), 3, 1000)
+ return service.post('/api/simulation/prepare', data)
}
/**
@@ -83,7 +83,7 @@ export const listSimulations = (projectId) => {
* @param {Object} data - { simulation_id, platform?, max_rounds?, enable_graph_memory_update? }
*/
export const startSimulation = (data) => {
- return requestWithRetry(() => service.post('/api/simulation/start', data), 3, 1000)
+ return service.post('/api/simulation/start', data)
}
/**
@@ -175,7 +175,7 @@ export const getEnvStatus = (data) => {
* @param {Object} data - { simulation_id, interviews: [{ agent_id, prompt }] }
*/
export const interviewAgents = (data) => {
- return requestWithRetry(() => service.post('/api/simulation/interview/batch', data), 3, 1000)
+ return service.post('/api/simulation/interview/batch', data)
}
/**
@@ -186,4 +186,3 @@ export const interviewAgents = (data) => {
export const getSimulationHistory = (limit = 20) => {
return service.get('/api/simulation/history', { params: { limit } })
}
-
diff --git a/frontend/src/components/Step3Simulation.vue b/frontend/src/components/Step3Simulation.vue
index 65e3f520..e9dd3dc3 100644
--- a/frontend/src/components/Step3Simulation.vue
+++ b/frontend/src/components/Step3Simulation.vue
@@ -515,22 +515,14 @@ const fetchRunStatus = async () => {
const isCompleted = data.runner_status === 'completed' || data.runner_status === 'stopped'
const isFailed = data.runner_status === 'failed'
- // 额外检查:如果后端还没来得及更新 runner_status,但平台已经报告完成
- // 通过检测 twitter_completed 和 reddit_completed 状态判断
- const platformsCompleted = checkPlatformsCompleted(data)
-
- // An explicit runner failure is authoritative. Platform completion flags
- // can be stale or partial when one subprocess exits successfully before
- // another subprocess fails, so inferred completion must not mask it.
+ // runner_status is authoritative because the backend only publishes a
+ // terminal state after the Zep ingestion barrier has completed.
if (isFailed) {
addLog(t('log.simFailed') + (data.error ? `: ${data.error}` : ''))
phase.value = 2
stopPolling()
emit('update-status', 'error')
- } else if (isCompleted || platformsCompleted) {
- if (platformsCompleted && !isCompleted) {
- addLog(t('log.allPlatformsCompleted'))
- }
+ } else if (isCompleted) {
addLog(t('log.simCompleted'))
phase.value = 2
stopPolling()
diff --git a/frontend/src/views/MainView.vue b/frontend/src/views/MainView.vue
index 513c70d8..cf600ff9 100644
--- a/frontend/src/views/MainView.vue
+++ b/frontend/src/views/MainView.vue
@@ -246,7 +246,6 @@ const loadProject = async () => {
} else if (res.data.status === 'graph_building' && res.data.graph_build_task_id) {
currentPhase.value = 1
startPollingTask(res.data.graph_build_task_id)
- startGraphPolling()
} else if (res.data.status === 'graph_completed' && res.data.graph_id) {
currentPhase.value = 2
await loadGraph(res.data.graph_id)
@@ -281,8 +280,18 @@ const startBuildGraph = async () => {
const res = await buildGraph({ project_id: currentProjectId.value })
if (res.success) {
+ if (res.data.reused && res.data.graph_id) {
+ currentPhase.value = 2
+ buildProgress.value = null
+ const projectRes = await getProject(currentProjectId.value)
+ if (projectRes.success) {
+ projectData.value = projectRes.data
+ }
+ await loadGraph(res.data.graph_id)
+ return
+ }
+
addLog(`Graph build task started. Task ID: ${res.data.task_id}`)
- startGraphPolling()
startPollingTask(res.data.task_id)
} else {
error.value = res.error
diff --git a/frontend/src/views/Process.vue b/frontend/src/views/Process.vue
index 2d2d3cc1..dcddeaec 100644
--- a/frontend/src/views/Process.vue
+++ b/frontend/src/views/Process.vue
@@ -686,14 +686,22 @@ const startBuildGraph = async () => {
const response = await buildGraph({ project_id: currentProjectId.value })
if (response.success) {
+ if (response.data.reused && response.data.graph_id) {
+ currentPhase.value = 2
+ buildProgress.value = null
+ const projectResponse = await getProject(currentProjectId.value)
+ if (projectResponse.success) {
+ projectData.value = projectResponse.data
+ }
+ await loadGraph(response.data.graph_id)
+ return
+ }
+
buildProgress.value.message = '图谱构建任务已启动...'
// 保存 task_id 用于轮询
const taskId = response.data.task_id
- // 启动图谱数据轮询(独立于任务状态轮询)
- startGraphPolling()
-
// 启动任务状态轮询
startPollingTask(taskId)
} else {
@@ -2065,4 +2073,4 @@ onUnmounted(() => {
display: none;
}
}
-</style>
\ No newline at end of file
+</style>