Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: add memory extract telemetry breakdown (#735)

zhoujiahuif467de4ed995
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4f24363a7..8feafacd7 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -15,18 +15,25 @@ Thank you for your interest in OpenViking! We welcome contributions of all kinds
- **Python**: 3.10+
- **Go**: 1.22+ (Required for building AGFS components from source)
+- **Rust**: 1.88+ (Required for source builds because the bundled `ov` CLI is built during packaging)
- **C++ Compiler**: GCC 9+ or Clang 11+ (Required for building core extensions, must support C++17)
- **CMake**: 3.12+
+#### Platform-Specific Native Build Tools
+
+- **Linux**: Install `build-essential`; some environments may also require `pkg-config`
+- **macOS**: Install Xcode Command Line Tools (`xcode-select --install`)
+- **Windows**: Install CMake and MinGW for local native builds
+
#### Supported Platforms (Pre-compiled Wheels)
OpenViking provides pre-compiled **Wheel** packages for the following environments:
- **Windows**: x86_64
- **macOS**: x86_64, arm64 (Apple Silicon)
-- **Linux**: x86_64 (manylinux)
+- **Linux**: x86_64, arm64 (manylinux)
-For other platforms (e.g., Linux ARM64, FreeBSD), the package will be automatically compiled from source during installation via `pip`. Ensure you have the [Prerequisites](#prerequisites) installed.
+For other platforms (e.g., FreeBSD), the package will be automatically compiled from source during installation via `pip`. Ensure you have the [Prerequisites](#prerequisites) installed.
### 1. Fork and Clone
@@ -49,15 +56,15 @@ source .venv/bin/activate # Linux/macOS
# or .venv\Scripts\activate # Windows
```
-#### Local Development & AGFS Compilation
+#### Local Development & Native Rebuilds
-OpenViking defaults to `binding-client` mode for AGFS, which requires a pre-built shared library. If you modify the **AGFS (Go)** code or **C++ extensions**, or if the pre-built library is not found, you need to re-compile and re-install them. Run the following command in the project root:
+OpenViking defaults to `binding-client` mode for AGFS, which requires pre-built native artifacts. If you modify the **AGFS (Go)** code, the bundled **Rust CLI**, or the **C++ extensions**, or if the pre-built artifacts are not found, you need to re-compile and re-install them. Run the following command in the project root:
```bash
uv pip install -e . --force-reinstall
```
-This command ensures that `setup.py` is re-executed, triggering the compilation of AGFS and C++ components.
+This command ensures that `setup.py` is re-executed, triggering rebuilds for AGFS, the bundled `ov` CLI, and the C++ components.
### 3. Configure Environment
@@ -108,7 +115,7 @@ asyncio.run(main())
The Rust CLI (`ov`) provides a high-performance command-line client for interacting with OpenViking Server.
-**Prerequisites**: Rust >= 1.88
+Even if you do not plan to use `ov` directly, the Rust toolchain is still required when building OpenViking from source because packaging also builds the bundled CLI binary.
```bash
# Build and install from source
@@ -134,53 +141,44 @@ openviking/
├── openviking/ # Python SDK
│ ├── async_client.py # AsyncOpenViking client
│ ├── sync_client.py # SyncOpenViking client
-│ │
-│ ├── core/ # Core data models
-│ │ ├── context.py # Context base class
-│ │ └── directories.py # Directory definitions
-│ │
-│ ├── parse/ # Resource parsers
-│ │ ├── parsers/ # Parser implementations
-│ │ ├── tree_builder.py
-│ │ └── registry.py
-│ │
+│ ├── client/ # Local and HTTP client implementations
+│ ├── console/ # Standalone console UI and proxy service
+│ ├── core/ # Core data models and directory abstractions
+│ ├── message/ # Session message and part models
+│ ├── models/ # Embedding and VLM backends
+│ ├── parse/ # Resource parsers and detectors
+│ ├── resource/ # Resource processing and watch management
│ ├── retrieve/ # Retrieval system
-│ │ ├── retriever.py # Main retriever
-│ │ ├── reranker.py # Reranking
-│ │ └── intent_analyzer.py
-│ │
-│ ├── session/ # Session management
-│ │ ├── session.py # Session core
-│ │ └── compressor.py # Compression
-│ │
│ ├── server/ # HTTP server
-│ │ ├── app.py # FastAPI app factory
-│ │ ├── bootstrap.py # Entry point (openviking-server)
-│ │ └── routers/ # API routers
-│ │
+│ ├── service/ # Shared service layer
+│ ├── session/ # Session management and compression
│ ├── storage/ # Storage layer
-│ │ ├── viking_fs.py # VikingFS
-│ │ └── vectordb/ # Vector database
-│ │
-│ ├── utils/ # Utilities
-│ │ └── config/ # Configuration
-│ │
+│ ├── telemetry/ # Operation telemetry
+│ ├── trace/ # Trace and runtime tracing helpers
+│ ├── utils/ # Utilities and configuration helpers
│ └── prompts/ # Prompt templates
│
├── crates/ # Rust components
│ └── ov_cli/ # Rust CLI client
-│ ├── src/ # CLI source code
-│ └── install.sh # Quick install script
+│ ├── src/ # CLI source code
+│ └── install.sh # Quick install script
│
├── src/ # C++ extensions (pybind11)
│
├── tests/ # Test suite
│ ├── client/ # Client tests
+│ ├── console/ # Console tests
+│ ├── core/ # Core logic tests
+│ ├── parse/ # Parser tests
+│ ├── resource/ # Resource processing tests
+│ ├── retrieve/ # Retrieval tests
│ ├── server/ # Server tests
+│ ├── service/ # Service layer tests
│ ├── session/ # Session tests
-│ ├── parse/ # Parser tests
+│ ├── storage/ # Storage tests
+│ ├── telemetry/ # Telemetry tests
│ ├── vectordb/ # Vector database tests
-│ └── integration/ # Integration tests
+│ └── integration/ # End-to-end tests
│
└── docs/ # Documentation
├── en/ # English docs
@@ -278,16 +276,16 @@ class TestAsyncOpenViking:
assert uninitialized_client._service is not None
await uninitialized_client.close()
- async def test_add_resource(self, client: AsyncOpenViking):
+ async def test_add_resource(self, client: AsyncOpenViking, sample_markdown_file):
result = await client.add_resource(
- "./test.md",
+ path=str(sample_markdown_file),
reason="test document"
)
- assert result["status"] == "success"
assert "root_uri" in result
+ assert result["root_uri"].startswith("viking://")
```
-Common fixtures are defined in `tests/conftest.py`, including `client` (initialized `AsyncOpenViking`), `uninitialized_client`, `temp_dir`, etc.
+Common fixtures are defined in `tests/conftest.py`, including `client` (initialized `AsyncOpenViking`), `uninitialized_client`, `temp_dir`, `sample_markdown_file`, and more.
---
@@ -424,10 +422,12 @@ We use **GitHub Actions** for Continuous Integration and Continuous Deployment.
| Event | Workflow | Description |
|-------|----------|-------------|
| **Pull Request** | `pr.yml` | Runs **Lint** (Ruff, Mypy) and **Test Lite** (Integration tests on Linux + Python 3.10). Provides fast feedback for contributors. (Displayed as **01. Pull Request Checks**) |
-| **Push to Main** | `ci.yml` | Runs **Test Full** (All OS: Linux/Win/Mac, All Py versions: 3.10-3.13) and **CodeQL** (Security scan). Ensures main branch stability. (Displayed as **02. Main Branch Checks**) |
+| **Push to Main** | `ci.yml` | Runs **Test Full** (All OS: Linux/Win/Mac, All Py versions: 3.10-3.14) and **CodeQL** (Security scan). Ensures main branch stability. (Displayed as **02. Main Branch Checks**) |
| **Release Published** | `release.yml` | Triggered when you create a Release on GitHub. Automatically builds source distribution and wheels, determines version from Git Tag, and publishes to **PyPI**. (Displayed as **03. Release**) |
| **Weekly Cron** | `schedule.yml` | Runs **CodeQL** security scan every Sunday. (Displayed as **04. Weekly Security Scan**) |
+Other repository workflows also exist for PR review automation, Docker image builds, and Rust CLI packaging.
+
### 2. Manual Trigger Workflows
Maintainers can manually trigger the following workflows from the "Actions" tab to perform specific tasks or debug issues.
@@ -445,11 +445,11 @@ Runs fast integration tests, supports custom matrix configuration.
* `python_json`: JSON string array of Python versions (e.g., `["3.10"]`).
#### C. Test Suite (Full) (`13. _Test Suite (Full)`)
-Runs the full test suite on all supported platforms (Linux/Mac/Win) and Python versions (3.10-3.13). Supports custom matrix configuration when triggered manually.
+Runs the full test suite on all supported platforms (Linux/Mac/Win) and Python versions (3.10-3.14). Supports custom matrix configuration when triggered manually.
* **Inputs**:
* `os_json`: List of OS to run on (Default: `["ubuntu-24.04", "macos-14", "windows-latest"]`).
- * `python_json`: List of Python versions (Default: `["3.10", "3.11", "3.12", "3.13"]`).
+ * `python_json`: List of Python versions (Default: `["3.10", "3.11", "3.12", "3.13", "3.14"]`).
#### D. Security Scan (`14. _CodeQL Scan`)
Runs CodeQL security analysis. No arguments required.
@@ -459,7 +459,7 @@ Builds Python wheel packages only, does not publish.
* **Inputs**:
* `os_json`: List of OS to build on (Default: `["ubuntu-24.04", "ubuntu-24.04-arm", "macos-14", "macos-15-intel", "windows-latest"]`).
- * `python_json`: List of Python versions (Default: `["3.10", "3.11", "3.12", "3.13"]`).
+ * `python_json`: List of Python versions (Default: `["3.10", "3.11", "3.12", "3.13", "3.14"]`).
* `build_sdist`: Whether to build source distribution (Default: `true`).
* `build_wheels`: Whether to build wheel distribution (Default: `true`).
diff --git a/CONTRIBUTING_CN.md b/CONTRIBUTING_CN.md
index 3e766bc0f..ad5b973ff 100644
--- a/CONTRIBUTING_CN.md
+++ b/CONTRIBUTING_CN.md
@@ -15,18 +15,25 @@
- **Python**: 3.10+
- **Go**: 1.22+ (从源码构建 AGFS 组件需要)
+- **Rust**: 1.88+(从源码构建需要;打包流程会同时构建内置 `ov` CLI)
- **C++ 编译器**: GCC 9+ 或 Clang 11+ (构建核心扩展需要,必须支持 C++17)
- **CMake**: 3.12+
+#### 平台相关的本地构建工具
+
+- **Linux**: 建议安装 `build-essential`;某些环境下还可能需要 `pkg-config`
+- **macOS**: 安装 Xcode Command Line Tools(`xcode-select --install`)
+- **Windows**: 本地原生构建建议安装 CMake 和 MinGW
+
#### 支持的平台 (预编译 Wheel 包)
OpenViking 为以下环境提供预编译的 **Wheel** 包,安装时无需本地编译:
- **Windows**: x86_64
- **macOS**: x86_64, arm64 (Apple Silicon)
-- **Linux**: x86_64 (manylinux)
+- **Linux**: x86_64, arm64 (manylinux)
-对于其他平台(如 Linux ARM64、FreeBSD 等),安装时 `pip` 将会自动从源码进行编译。请确保已安装上述[前置要求](#前置要求)中的工具。
+对于其他平台(如 FreeBSD 等),安装时 `pip` 将会自动从源码进行编译。请确保已安装上述[前置要求](#前置要求)中的工具。
### 1. Fork 并克隆
@@ -49,15 +56,15 @@ source .venv/bin/activate # Linux/macOS
# 或者 .venv\Scripts\activate # Windows
```
-#### 本地开发与 AGFS 编译
+#### 本地开发与原生组件重建
-OpenViking 默认使用 `binding-client` 模式,这需要一个预先构建的共享库。如果你修改了 **AGFS (Go)** 代码或 **C++ 扩展**,或者预编译库未找到,你需要重新编译并安装它们,以使更改在本地环境中生效。在项目根目录下运行以下命令:
+OpenViking 默认使用 `binding-client` 模式,这依赖预先构建的原生产物。如果你修改了 **AGFS (Go)** 代码、内置的 **Rust CLI**,或 **C++ 扩展**,或者这些预编译产物未找到,你需要重新编译并安装它们,以使更改在本地环境中生效。在项目根目录下运行以下命令:
```bash
uv pip install -e . --force-reinstall
```
-该命令会强制重新执行 `setup.py`,触发 AGFS 和 C++ 组件的编译与安装。
+该命令会强制重新执行 `setup.py`,触发 AGFS、内置 `ov` CLI 和 C++ 组件的重新编译与安装。
### 3. 配置环境
@@ -108,7 +115,7 @@ asyncio.run(main())
Rust CLI (`ov`) 提供高性能的命令行客户端,用于连接 OpenViking Server 进行操作。
-**前置要求**:Rust >= 1.88
+即使你不打算直接使用 `ov`,只要走 OpenViking 的源码构建流程,Rust 工具链仍然是必需的,因为打包过程也会构建内置 CLI 二进制。
```bash
# 从源码编译安装
@@ -134,53 +141,44 @@ openviking/
├── openviking/ # Python SDK
│ ├── async_client.py # AsyncOpenViking 客户端
│ ├── sync_client.py # SyncOpenViking 客户端
-│ │
-│ ├── core/ # 核心数据模型
-│ │ ├── context.py # 上下文基类
-│ │ └── directories.py # 目录定义
-│ │
-│ ├── parse/ # 资源解析器
-│ │ ├── parsers/ # 解析器实现
-│ │ ├── tree_builder.py
-│ │ └── registry.py
-│ │
+│ ├── client/ # 本地与 HTTP 客户端实现
+│ ├── console/ # 独立 console UI 与代理服务
+│ ├── core/ # 核心数据模型与目录抽象
+│ ├── message/ # 会话消息与 part 模型
+│ ├── models/ # Embedding 与 VLM 后端
+│ ├── parse/ # 资源解析器与检测器
+│ ├── resource/ # 资源处理与 watch 管理
│ ├── retrieve/ # 检索系统
-│ │ ├── retriever.py # 主检索器
-│ │ ├── reranker.py # 重排序
-│ │ └── intent_analyzer.py
-│ │
-│ ├── session/ # 会话管理
-│ │ ├── session.py # 会话核心
-│ │ └── compressor.py # 压缩
-│ │
│ ├── server/ # HTTP 服务端
-│ │ ├── app.py # FastAPI 应用工厂
-│ │ ├── bootstrap.py # 启动入口 (openviking-server)
-│ │ └── routers/ # API 路由
-│ │
+│ ├── service/ # 共享 service 层
+│ ├── session/ # 会话管理与压缩
│ ├── storage/ # 存储层
-│ │ ├── viking_fs.py # VikingFS
-│ │ └── vectordb/ # 向量数据库
-│ │
-│ ├── utils/ # 工具类
-│ │ └── config/ # 配置
-│ │
+│ ├── telemetry/ # 操作级 telemetry
+│ ├── trace/ # trace 与运行时跟踪辅助
+│ ├── utils/ # 工具类与配置辅助
│ └── prompts/ # 提示词模板
│
├── crates/ # Rust 组件
│ └── ov_cli/ # Rust CLI 客户端
-│ ├── src/ # CLI 源码
-│ └── install.sh # 一键安装脚本
+│ ├── src/ # CLI 源码
+│ └── install.sh # 一键安装脚本
│
├── src/ # C++ 扩展 (pybind11)
│
├── tests/ # 测试套件
│ ├── client/ # 客户端测试
+│ ├── console/ # Console 测试
+│ ├── core/ # 核心逻辑测试
+│ ├── parse/ # 解析器测试
+│ ├── resource/ # 资源处理测试
+│ ├── retrieve/ # 检索测试
│ ├── server/ # 服务端测试
+│ ├── service/ # Service 层测试
│ ├── session/ # 会话测试
-│ ├── parse/ # 解析器测试
+│ ├── storage/ # 存储测试
+│ ├── telemetry/ # Telemetry 测试
│ ├── vectordb/ # 向量数据库测试
-│ └── integration/ # 集成测试
+│ └── integration/ # 端到端测试
│
└── docs/ # 文档
├── en/ # 英文文档
@@ -278,16 +276,16 @@ class TestAsyncOpenViking:
assert uninitialized_client._service is not None
await uninitialized_client.close()
- async def test_add_resource(self, client: AsyncOpenViking):
+ async def test_add_resource(self, client: AsyncOpenViking, sample_markdown_file):
result = await client.add_resource(
- "./test.md",
+ path=str(sample_markdown_file),
reason="test document"
)
- assert result["status"] == "success"
assert "root_uri" in result
+ assert result["root_uri"].startswith("viking://")
```
-常用 fixture 定义在 `tests/conftest.py` 中,包括 `client`(已初始化的 `AsyncOpenViking`)、`uninitialized_client`、`temp_dir` 等。
+常用 fixture 定义在 `tests/conftest.py` 中,包括 `client`(已初始化的 `AsyncOpenViking`)、`uninitialized_client`、`temp_dir`、`sample_markdown_file` 等。
---
@@ -424,10 +422,12 @@ git commit -m "refactor(storage): simplify interface methods"
| 事件 | 工作流 | 描述 |
|-------|----------|-------------|
| **Pull Request** | `pr.yml` | 运行 **Lint** (Ruff, Mypy) 和 **Test Lite** (Linux + Python 3.10 上的集成测试)。为贡献者提供快速反馈。(显示为 **01. Pull Request Checks**) |
-| **Push to Main** | `ci.yml` | 运行 **Test Full** (所有操作系统:Linux/Win/Mac,所有 Py版本:3.10-3.13) 和 **CodeQL** (安全扫描)。确保主分支稳定性。(显示为 **02. Main Branch Checks**) |
+| **Push to Main** | `ci.yml` | 运行 **Test Full** (所有操作系统:Linux/Win/Mac,所有 Py版本:3.10-3.14) 和 **CodeQL** (安全扫描)。确保主分支稳定性。(显示为 **02. Main Branch Checks**) |
| **Release Published** | `release.yml` | 当您在 GitHub 上创建 Release 时触发。自动构建源码包和 wheel 包,基于 Git Tag 确定版本号,并发布到 **PyPI**。(显示为 **03. Release**) |
| **Weekly Cron** | `schedule.yml` | 每周日运行 **CodeQL** 安全扫描。(显示为 **04. Weekly Security Scan**) |
+仓库中另外还有用于 PR review 自动化、Docker 镜像构建和 Rust CLI 打包的工作流。
+
### 2. 手动触发工作流
维护者可以从 "Actions" 选项卡手动触发以下工作流,以执行特定任务或调试问题。
@@ -445,11 +445,11 @@ git commit -m "refactor(storage): simplify interface methods"
* `python_json`: Python 版本列表的 JSON 字符串数组 (例如 `["3.10"]`)。
#### C. 完整测试 (`13. _Test Suite (Full)`)
-在所有支持的平台 (Linux/Mac/Win) 和 Python 版本 (3.10-3.13) 上运行完整的测试套件。手动触发时支持自定义矩阵配置。
+在所有支持的平台 (Linux/Mac/Win) 和 Python 版本 (3.10-3.14) 上运行完整的测试套件。手动触发时支持自定义矩阵配置。
* **Inputs**:
* `os_json`: 操作系统列表 (默认: `["ubuntu-24.04", "macos-14", "windows-latest"]`)。
- * `python_json`: Python 版本列表 (默认: `["3.10", "3.11", "3.12", "3.13"]`)。
+ * `python_json`: Python 版本列表 (默认: `["3.10", "3.11", "3.12", "3.13", "3.14"]`)。
#### D. 安全扫描 (`14. _CodeQL Scan`)
运行 CodeQL 安全分析。无需参数。
@@ -459,7 +459,7 @@ git commit -m "refactor(storage): simplify interface methods"
* **Inputs**:
* `os_json`: 操作系统列表 (默认: `["ubuntu-24.04", "ubuntu-24.04-arm", "macos-14", "macos-15-intel", "windows-latest"]`)。
- * `python_json`: Python 版本列表 (默认: `["3.10", "3.11", "3.12", "3.13"]`)。
+ * `python_json`: Python 版本列表 (默认: `["3.10", "3.11", "3.12", "3.13", "3.14"]`)。
* `build_sdist`: 是否构建源码包 (默认: `true`)。
* `build_wheels`: 是否构建 Wheel 包 (默认: `true`)。
diff --git a/CONTRIBUTING_JA.md b/CONTRIBUTING_JA.md
index 6f550f168..70fc9100e 100644
--- a/CONTRIBUTING_JA.md
+++ b/CONTRIBUTING_JA.md
@@ -15,18 +15,25 @@ OpenVikingに興味をお持ちいただきありがとうございます!あ
- **Python**: 3.10以上
- **Go**: 1.22以上(AGFSコンポーネントのソースビルドに必要)
+- **Rust**: 1.88以上(ソースビルド時に同梱の `ov` CLI もビルドされるため必須)
- **C++コンパイラ**: GCC 9以上 または Clang 11以上(コア拡張のビルドに必要、C++17サポートが必須)
- **CMake**: 3.12以上
+#### プラットフォーム別のネイティブビルドツール
+
+- **Linux**: `build-essential` の導入を推奨。環境によっては `pkg-config` も必要です
+- **macOS**: Xcode Command Line Tools をインストール(`xcode-select --install`)
+- **Windows**: ローカルのネイティブビルドには CMake と MinGW を推奨
+
#### サポートされているプラットフォーム(プリコンパイル済みWheel)
OpenVikingは以下の環境向けにプリコンパイル済み**Wheel**パッケージを提供しています:
- **Windows**: x86_64
- **macOS**: x86_64、arm64(Apple Silicon)
-- **Linux**: x86_64(manylinux)
+- **Linux**: x86_64、arm64(manylinux)
-その他のプラットフォーム(例:Linux ARM64、FreeBSD)では、`pip`によるインストール時にソースから自動コンパイルされます。[前提条件](#前提条件)がインストールされていることを確認してください。
+その他のプラットフォーム(例:FreeBSD)では、`pip`によるインストール時にソースから自動コンパイルされます。[前提条件](#前提条件)がインストールされていることを確認してください。
### 1. フォークとクローン
@@ -49,15 +56,15 @@ source .venv/bin/activate # Linux/macOS
# または .venv\Scripts\activate # Windows
```
-#### ローカル開発とAGFSコンパイル
+#### ローカル開発とネイティブコンポーネントの再ビルド
-OpenVikingはAGFSに対してデフォルトで`binding-client`モードを使用し、プリビルドされた共有ライブラリが必要です。**AGFS(Go)**コードや**C++拡張**を変更した場合、またはプリビルドライブラリが見つからない場合は、再コンパイルと再インストールが必要です。プロジェクトルートで以下のコマンドを実行してください:
+OpenVikingはAGFSに対してデフォルトで`binding-client`モードを使用し、事前にビルドされたネイティブ成果物を利用します。**AGFS(Go)**コード、同梱の**Rust CLI**、または**C++拡張**を変更した場合や、プリビルド成果物が見つからない場合は、再コンパイルと再インストールが必要です。プロジェクトルートで以下のコマンドを実行してください:
```bash
uv pip install -e . --force-reinstall
```
-このコマンドにより`setup.py`が再実行され、AGFSおよびC++コンポーネントのコンパイルがトリガーされます。
+このコマンドにより`setup.py`が再実行され、AGFS、同梱 `ov` CLI、C++コンポーネントの再ビルドがトリガーされます。
### 3. 環境設定
@@ -108,6 +115,8 @@ asyncio.run(main())
Rust CLI(`ov`)は、OpenViking Serverとやり取りするための高性能コマンドラインクライアントを提供します。
+`ov` を直接使わない場合でも、OpenViking をソースからビルドするなら Rust ツールチェーンは必要です。パッケージング時に同梱 CLI バイナリも一緒にビルドされるためです。
+
**前提条件**: Rust >= 1.88
```bash
@@ -134,53 +143,44 @@ openviking/
├── openviking/ # Python SDK
│ ├── async_client.py # AsyncOpenVikingクライアント
│ ├── sync_client.py # SyncOpenVikingクライアント
-│ │
-│ ├── core/ # コアデータモデル
-│ │ ├── context.py # Contextベースクラス
-│ │ └── directories.py # ディレクトリ定義
-│ │
-│ ├── parse/ # リソースパーサー
-│ │ ├── parsers/ # パーサー実装
-│ │ ├── tree_builder.py
-│ │ └── registry.py
-│ │
+│ ├── client/ # ローカル / HTTP クライアント実装
+│ ├── console/ # スタンドアロン console UI とプロキシサービス
+│ ├── core/ # コアデータモデルとディレクトリ抽象
+│ ├── message/ # セッションメッセージと part モデル
+│ ├── models/ # Embedding / VLM バックエンド
+│ ├── parse/ # リソースパーサーと検出器
+│ ├── resource/ # リソース処理と watch 管理
│ ├── retrieve/ # 検索システム
-│ │ ├── retriever.py # メインリトリーバー
-│ │ ├── reranker.py # リランキング
-│ │ └── intent_analyzer.py
-│ │
-│ ├── session/ # セッション管理
-│ │ ├── session.py # セッションコア
-│ │ └── compressor.py # 圧縮
-│ │
│ ├── server/ # HTTPサーバー
-│ │ ├── app.py # FastAPIアプリファクトリ
-│ │ ├── bootstrap.py # エントリポイント(openviking-server)
-│ │ └── routers/ # APIルーター
-│ │
+│ ├── service/ # 共通 service レイヤー
+│ ├── session/ # セッション管理と圧縮
│ ├── storage/ # ストレージレイヤー
-│ │ ├── viking_fs.py # VikingFS
-│ │ └── vectordb/ # ベクトルデータベース
-│ │
-│ ├── utils/ # ユーティリティ
-│ │ └── config/ # 設定
-│ │
+│ ├── telemetry/ # オペレーション telemetry
+│ ├── trace/ # trace とランタイム追跡補助
+│ ├── utils/ # ユーティリティと設定補助
│ └── prompts/ # プロンプトテンプレート
│
├── crates/ # Rustコンポーネント
│ └── ov_cli/ # Rust CLIクライアント
-│ ├── src/ # CLIソースコード
-│ └── install.sh # クイックインストールスクリプト
+│ ├── src/ # CLIソースコード
+│ └── install.sh # クイックインストールスクリプト
│
├── src/ # C++拡張(pybind11)
│
├── tests/ # テストスイート
│ ├── client/ # クライアントテスト
+│ ├── console/ # Console テスト
+│ ├── core/ # コアロジックテスト
+│ ├── parse/ # パーサーテスト
+│ ├── resource/ # リソース処理テスト
+│ ├── retrieve/ # 検索テスト
│ ├── server/ # サーバーテスト
+│ ├── service/ # Service レイヤーテスト
│ ├── session/ # セッションテスト
-│ ├── parse/ # パーサーテスト
+│ ├── storage/ # ストレージテスト
+│ ├── telemetry/ # Telemetry テスト
│ ├── vectordb/ # ベクトルデータベーステスト
-│ └── integration/ # 統合テスト
+│ └── integration/ # E2E テスト
│
└── docs/ # ドキュメント
├── en/ # 英語ドキュメント
@@ -278,16 +278,16 @@ class TestAsyncOpenViking:
assert uninitialized_client._service is not None
await uninitialized_client.close()
- async def test_add_resource(self, client: AsyncOpenViking):
+ async def test_add_resource(self, client: AsyncOpenViking, sample_markdown_file):
result = await client.add_resource(
- "./test.md",
+ path=str(sample_markdown_file),
reason="test document"
)
- assert result["status"] == "success"
assert "root_uri" in result
+ assert result["root_uri"].startswith("viking://")
```
-共通フィクスチャは`tests/conftest.py`に定義されており、`client`(初期化済み`AsyncOpenViking`)、`uninitialized_client`、`temp_dir`などが含まれます。
+共通フィクスチャは`tests/conftest.py`に定義されており、`client`(初期化済み`AsyncOpenViking`)、`uninitialized_client`、`temp_dir`、`sample_markdown_file` などが含まれます。
---
@@ -424,10 +424,12 @@ git commit -m "refactor(storage): simplify interface methods"
| イベント | ワークフロー | 説明 |
|-------|----------|-------------|
| **プルリクエスト** | `pr.yml` | **Lint**(Ruff、Mypy)と**Test Lite**(Linux + Python 3.10での統合テスト)を実行。コントリビューターに迅速なフィードバックを提供。(**01. Pull Request Checks**として表示) |
-| **mainへのプッシュ** | `ci.yml` | **Test Full**(全OS:Linux/Win/Mac、全Pyバージョン:3.10-3.13)と**CodeQL**(セキュリティスキャン)を実行。mainブランチの安定性を保証。(**02. Main Branch Checks**として表示) |
+| **mainへのプッシュ** | `ci.yml` | **Test Full**(全OS:Linux/Win/Mac、全Pyバージョン:3.10-3.14)と**CodeQL**(セキュリティスキャン)を実行。mainブランチの安定性を保証。(**02. Main Branch Checks**として表示) |
| **リリース公開** | `release.yml` | GitHubでリリースを作成すると発動。自動的にソースディストリビューションとwheelをビルドし、Gitタグからバージョンを判定して**PyPI**に公開。(**03. Release**として表示) |
| **週次Cron** | `schedule.yml` | 毎週日曜日に**CodeQL**セキュリティスキャンを実行。(**04. Weekly Security Scan**として表示) |
+このほか、PR review の自動化、Docker イメージのビルド、Rust CLI のパッケージング用ワークフローも用意されています。
+
### 2. 手動トリガーワークフロー
メンテナーは「Actions」タブから以下のワークフローを手動でトリガーして、特定のタスクを実行したり問題をデバッグしたりできます。
@@ -445,11 +447,11 @@ git commit -m "refactor(storage): simplify interface methods"
* `python_json`: Pythonバージョンの JSON文字列配列(例:`["3.10"]`)。
#### C. テストスイート(Full)(`13. _Test Suite (Full)`)
-サポートされているすべてのプラットフォーム(Linux/Mac/Win)とPythonバージョン(3.10-3.13)で完全なテストスイートを実行。手動トリガー時にカスタムマトリックス設定をサポートします。
+サポートされているすべてのプラットフォーム(Linux/Mac/Win)とPythonバージョン(3.10-3.14)で完全なテストスイートを実行。手動トリガー時にカスタムマトリックス設定をサポートします。
* **入力**:
* `os_json`: 実行するOSのリスト(デフォルト:`["ubuntu-24.04", "macos-14", "windows-latest"]`)。
- * `python_json`: Pythonバージョンのリスト(デフォルト:`["3.10", "3.11", "3.12", "3.13"]`)。
+ * `python_json`: Pythonバージョンのリスト(デフォルト:`["3.10", "3.11", "3.12", "3.13", "3.14"]`)。
#### D. セキュリティスキャン (`14. _CodeQL Scan`)
CodeQLセキュリティ分析を実行。引数は不要です。
@@ -459,7 +461,7 @@ Pythonのwheelパッケージのみをビルドし、公開はしません。
* **入力**:
* `os_json`: ビルドするOSのリスト(デフォルト:`["ubuntu-24.04", "ubuntu-24.04-arm", "macos-14", "macos-15-intel", "windows-latest"]`)。
- * `python_json`: Pythonバージョンのリスト(デフォルト:`["3.10", "3.11", "3.12", "3.13"]`)。
+ * `python_json`: Pythonバージョンのリスト(デフォルト:`["3.10", "3.11", "3.12", "3.13", "3.14"]`)。
* `build_sdist`: ソースディストリビューションをビルドするか(デフォルト:`true`)。
* `build_wheels`: wheelディストリビューションをビルドするか(デフォルト:`true`)。
diff --git a/docs/design/operation_telemetry.md b/docs/design/operation_telemetry.md
deleted file mode 100644
index d20fd00eb..000000000
--- a/docs/design/operation_telemetry.md
+++ /dev/null
@@ -1,520 +0,0 @@
-# 操作级 Telemetry 设计
-
-## 1. 背景与目标
-
-OpenViking 需要一套统一的 telemetry 机制,用来描述一次操作在执行过程中的关键观测信息。当前已经落地的是操作级 telemetry,主要覆盖:
-
-- 请求耗时统计
-- token 消耗统计
-- 检索、队列、内存提取等阶段的摘要指标
-
-这里统一使用 `telemetry`,而不是 `trace`,原因是这套抽象未来不只服务于“单次操作链路”,还要能承载非操作级数据,例如:
-
-- 服务整体 token 消耗
-- 各类后端能力的延迟与错误率
-- 存储、向量库、队列等组件级指标
-- 基于 OpenTelemetry 的 exporter / backend 对接
-
-当前实现只对“操作级 telemetry”提供正式接口,但抽象命名和结构已经为后续扩展预留空间。
-
-## 2. 设计原则
-
-### 2.1 详细信息显式按需返回
-
-详细 telemetry 由调用方通过 `telemetry` 参数显式请求,当前对外协议只返回结构化 summary,不返回事件流。
-
-### 2.3 字段名直接面向用户
-
-内部打点名与对外 summary 字段名保持一致,避免额外的“内部名 -> 外部名”转换层。
-
-### 2.4 缺失分组不返回
-
-如果某类操作天然不会产出某个 summary 分组,则该分组直接省略,不返回空对象或全 `null` 字段。
-
-例如:
-
-- `resources.add_resource` 不一定有 `memory`
-- `session.commit` 一般没有 `semantic_nodes`
-- 某些操作没有向量检索,就不返回 `vector`
-
-## 3. 当前支持范围
-
-### 3.1 HTTP 接口
-
-当前已接入 operation telemetry 的接口:
-
-- `POST /api/v1/search/find`
-- `POST /api/v1/search/search`
-- `POST /api/v1/resources`
-- `POST /api/v1/skills`
-- `POST /api/v1/sessions/{session_id}/commit`
-
-说明:
-
-- `session.commit` 仅在 `wait=true` 的同步模式下支持返回 telemetry
-- `wait=false` 的异步任务模式当前不支持 telemetry,请求时会返回 `INVALID_ARGUMENT`
-
-### 3.2 SDK 接口
-
-当前已接入 operation telemetry 的 SDK 方法:
-
-- `add_resource`
-- `add_skill`
-- `find`
-- `search`
-- `commit_session`
-
-本地嵌入式 client 和 HTTP client 都遵循同一套 telemetry 请求语义。
-
-## 4. 响应模型
-
-服务端仍使用统一响应包裹结构:
-
-```json
-{
- "status": "ok",
- "result": { "...": "..." },
- "time": 0.031,
- "telemetry": {
- "id": "tm_9f6f4d6b0d0c4f4d93ce5adf82e71c18",
- "summary": {
- "operation": "search.find",
- "status": "ok",
- "duration_ms": 31.224,
- "tokens": {
- "total": 24,
- "llm": {
- "input": 12,
- "output": 6,
- "total": 18
- },
- "embedding": {
- "total": 6
- }
- },
- "vector": {
- "searches": 3,
- "scored": 26,
- "passed": 8,
- "returned": 5,
- "scanned": 26,
- "scan_reason": ""
- }
- }
- }
-}
-```
-
-说明:
-
-- `telemetry` 只在调用方显式请求时返回
-- `telemetry.id` 是不透明标识,只用于关联,不要求调用方解析语义
-
-## 5. telemetry 请求语义
-
-`telemetry` 字段支持两种形态:
-
-### 5.1 布尔形态
-
-```json
-{
- "telemetry": true
-}
-```
-
-语义:
-
-- 返回 `telemetry.id + telemetry.summary`
-
-### 5.2 对象形态
-
-```json
-{
- "telemetry": {
- "summary": true
- }
-}
-```
-
-语义:
-
-- `summary` 默认值为 `true`
-- 适合只看结构化摘要
-
-当前支持的合法组合如下:
-
-| 请求值 | 语义 |
-| --- | --- |
-| `false` | 不返回 `telemetry` |
-| `true` | 返回 `id + summary` |
-| `{"summary": true}` | 返回 `id + summary` |
-| `{"summary": false}` | 不返回 `telemetry` |
-
-以下请求非法:
-
-```json
-{
- "telemetry": {
- "events": true
- }
-}
-```
-
-原因是当前对外 telemetry 已收敛为 summary-only,不再接受事件流选择参数。
-
-## 6. telemetry 的职责划分
-
-### 6.1 `telemetry.summary`
-
-`summary` 是结构化的操作摘要,用于:
-
-- 调试
-- 排障
-- 离线分析
-- 上报到外部观测系统
-
-当前 summary 的核心字段包括:
-
-- `operation`
-- `status`
-- `duration_ms`
-- `tokens`
-- `queue`
-- `vector`
-- `semantic_nodes`
-- `memory`
-- `errors`
-
-其中:
-
-- `tokens` 始终存在
-- 其余分组按是否有产出决定是否返回
-
-### 6.3 `telemetry.id`
-
-`telemetry.id` 是请求级关联标识,用于把一次操作的 summary 与内部异步链路统计关联起来。
-
-## 7. summary 字段约定
-
-### 7.1 顶层公共字段
-
-所有 summary 至少包含:
-
-- `operation`
-- `status`
-- `duration_ms`
-- `tokens`
-
-### 7.2 tokens
-
-示例:
-
-```json
-{
- "tokens": {
- "total": 19,
- "llm": {
- "input": 11,
- "output": 7,
- "total": 18
- },
- "embedding": {
- "total": 1
- }
- }
-}
-```
-
-说明:
-
-- `llm` 统计输入、输出与总量
-- `embedding` 当前只统计总量
-
-### 7.3 queue
-
-队列相关摘要示例:
-
-```json
-{
- "queue": {
- "semantic": {
- "processed": 1,
- "error_count": 0
- },
- "embedding": {
- "processed": 1,
- "error_count": 0
- }
- }
-}
-```
-
-### 7.4 vector
-
-向量检索摘要示例:
-
-```json
-{
- "vector": {
- "searches": 2,
- "scored": 5,
- "passed": 3,
- "returned": 2,
- "scanned": 5,
- "scan_reason": ""
- }
-}
-```
-
-### 7.5 semantic_nodes
-
-语义检索 DAG / 节点级摘要示例:
-
-```json
-{
- "semantic_nodes": {
- "total": 4,
- "done": 3,
- "pending": 1,
- "running": 0
- }
-}
-```
-
-### 7.6 memory
-
-会话提交等内存提取类操作示例:
-
-```json
-{
- "memory": {
- "extracted": 4
- }
-}
-```
-
-### 7.7 errors
-
-发生错误时可返回:
-
-```json
-{
- "errors": {
- "stage": "resource_processor.parse",
- "error_code": "PROCESSING_ERROR",
- "message": "..."
- }
-}
-```
-
-无错误时,该分组可以省略。
-
-## 8. 缺失字段裁剪策略
-
-summary 采用“按分组裁剪”的策略,而不是固定返回整套字段。
-
-这样做有几个直接收益:
-
-- 避免返回大量与当前操作无关的空字段
-- 降低调用方理解成本
-- 更适合未来扩展新的 telemetry 分组
-
-示例:
-
-### 8.1 `resources.add_resource`
-
-可能返回:
-
-```json
-{
- "operation": "resources.add_resource",
- "status": "ok",
- "duration_ms": 152.3,
- "tokens": { "...": "..." },
- "semantic_nodes": { "...": "..." },
- "queue": { "...": "..." }
-}
-```
-
-这里不应强行返回 `memory`。
-
-### 8.2 `session.commit`
-
-可能返回:
-
-```json
-{
- "operation": "session.commit",
- "status": "ok",
- "duration_ms": 48.1,
- "tokens": { "...": "..." },
- "memory": {
- "extracted": 4
- }
-}
-```
-
-这里不应强行返回 `semantic_nodes`。
-
-## 9. 成本模型
-
-当前 collector 只采集 summary 所需的数据:
-
-- 采集 counters / gauges
-- 记录 error 状态
-- 构造最终 summary
-- 不保留事件列表
-
-## 10. 实现结构
-
-### 10.1 核心类型
-
-核心实现位于:
-
-- `openviking/telemetry/operation.py`
-- `openviking/telemetry/request.py`
-- `openviking/telemetry/context.py`
-- `openviking/telemetry/registry.py`
-
-主要对象包括:
-
-- `OperationTelemetry`
-- `TelemetrySnapshot`
-- `TelemetrySelection`
-
-### 10.2 请求解析
-
-`openviking/telemetry/request.py` 负责统一解析 `telemetry` 请求参数:
-
-- 支持 `bool | object`
-- 归一化为 `TelemetrySelection`
-- 校验非法字段,例如 `events`
-
-这样 server、local client、HTTP client 都共享同一套语义。
-
-### 10.3 服务端集成
-
-`openviking/server/telemetry.py` 负责:
-
-- 根据请求创建 collector
-- 根据 selection 决定是否附带 `summary`
-
-router 层的职责是:
-
-1. 创建 collector
-2. 绑定 operation 上下文
-3. 执行实际业务逻辑
-4. 按请求返回 `telemetry`
-
-### 10.4 本地与 HTTP client
-
-本地 client 和 HTTP client 都暴露同样的 `telemetry` 参数语义:
-
-```python
-await client.find("memory dedup", telemetry=True)
-await client.find("memory dedup", telemetry={"summary": True})
-```
-
-其中:
-
-- local client 在本地生成 telemetry 并拼回结果
-- HTTP client 负责参数校验并透传给服务端
-
-## 11. 异步链路与跨组件聚合
-
-当前 operation telemetry 不只覆盖同步请求栈,也支持部分异步处理链路的数据回流。
-
-典型场景包括:
-
-- 请求线程触发语义队列处理
-- 请求线程触发 embedding 处理
-- 后台处理线程继续向同一个 operation collector 记录指标
-
-实现方式是:
-
-- collector 生成 `telemetry.id`
-- 后续消息携带该 `id`
-- 后台组件通过 registry 找回原 collector
-- 在新的执行上下文中重新绑定 collector
-
-这样一次操作的最终 summary 可以覆盖:
-
-- 请求入口逻辑
-- 检索过程
-- embedding 处理
-- semantic queue 处理
-- memory 提取结果
-
-## 12. 与 OpenTelemetry 的关系
-
-当前方案不是直接把 OpenTelemetry 暴露为业务接口,而是先定义 OpenViking 自己的 telemetry 抽象。
-
-这样做的好处是:
-
-- 对调用方暴露稳定、简单的产品接口
-- 不把业务接口和具体观测框架强绑定
-- 后续可以新增 OpenTelemetry backend,而不影响现有 SDK / HTTP 语义
-
-可以把 OpenTelemetry 看作未来的一种底层实现或导出方式,而不是当前对外协议本身。
-
-## 13. 未来扩展方向
-
-当前文档描述的是 operation telemetry,但未来需要兼容更广义的 telemetry 数据源。
-
-推荐的扩展方向:
-
-- 服务级 token 消耗聚合
-- 存储、向量库、模型服务的接口耗时
-- 队列吞吐、失败率、积压长度
-- 与 OpenTelemetry exporter 的桥接
-- 更长期的指标聚合、采样和导出
-
-这些扩展不要求沿用完全相同的 summary schema,但应复用统一的 telemetry 抽象和运行时。
-
-## 14. 使用示例
-
-### 14.1 返回 telemetry summary
-
-```bash
-curl -X POST http://localhost:8080/api/v1/search/find \
- -H 'Content-Type: application/json' \
- -d '{
- "query": "memory dedup",
- "limit": 5,
- "telemetry": true
- }'
-```
-
-### 14.2 只返回 summary
-
-```bash
-curl -X POST http://localhost:8080/api/v1/search/find \
- -H 'Content-Type: application/json' \
- -d '{
- "query": "memory dedup",
- "limit": 5,
- "telemetry": {
- "summary": true
- }
- }'
-```
-
-### 14.4 Python SDK
-
-```python
-result = await client.find("memory dedup", telemetry={"summary": True})
-
-print(result.telemetry["summary"]["tokens"]["total"])
-```
-
-## 15. 新接口接入规范
-
-新接口如果需要接入 operation telemetry,建议遵循以下规则:
-
-1. 为该操作创建 `OperationTelemetry` collector。
-2. 用上下文绑定覆盖整个操作生命周期。
-3. 在内部关键阶段记录 counters、gauges 和错误状态。
-4. 仅在调用方请求时返回 `telemetry`。
-5. summary 只返回本次操作真实产出的分组。
-
-这样可以保持默认低成本,同时为调用方提供稳定、可分析的结构化摘要。
diff --git a/docs/en/guides/07-operation-telemetry.md b/docs/en/guides/07-operation-telemetry.md
new file mode 100644
index 000000000..27c452c47
--- /dev/null
+++ b/docs/en/guides/07-operation-telemetry.md
@@ -0,0 +1,302 @@
+# Operation Telemetry
+
+Operation telemetry lets you ask OpenViking to return a compact summary of what happened during a request, such as duration, token usage, vector retrieval activity, queue progress, and resource-processing stages.
+
+Use it when you want to:
+
+- debug a slow or unexpected request
+- inspect token or retrieval behavior
+- capture structured execution data in your own logs or observability pipeline
+
+## How it works
+
+Telemetry is opt-in. OpenViking only returns a top-level `telemetry` object when you request it.
+
+Typical response shape:
+
+```json
+{
+ "status": "ok",
+ "result": {"...": "..."},
+ "telemetry": {
+ "id": "tm_xxx",
+ "summary": {
+ "operation": "search.find",
+ "status": "ok",
+ "duration_ms": 31.2,
+ "tokens": {
+ "total": 24,
+ "llm": {
+ "input": 12,
+ "output": 6,
+ "total": 18
+ }
+ },
+ "vector": {
+ "searches": 3,
+ "scored": 26,
+ "passed": 8,
+ "returned": 5
+ }
+ }
+ }
+}
+```
+
+Notes:
+
+- `telemetry.id` is an opaque correlation id
+- `telemetry.summary` is the structured payload intended for users
+- summary groups appear only when the operation produced them
+- numeric `0` values are omitted from the response
+
+## Supported operations
+
+### HTTP API
+
+Operation telemetry is currently available on these endpoints:
+
+- `POST /api/v1/search/find`
+- `POST /api/v1/search/search`
+- `POST /api/v1/resources/temp_upload`
+- `POST /api/v1/resources`
+- `POST /api/v1/skills`
+- `POST /api/v1/sessions/{session_id}/commit`
+
+### Python SDK
+
+The same telemetry model is available from the Python clients for:
+
+- `add_resource(...)`
+- `add_skill(...)`
+- `find(...)`
+- `search(...)`
+- `commit_session(...)`
+- `Session.commit(...)`
+
+## Requesting telemetry
+
+### JSON APIs
+
+For JSON request bodies, `telemetry` supports these forms:
+
+```json
+{"telemetry": true}
+```
+
+```json
+{"telemetry": {"summary": true}}
+```
+
+`true` and `{"summary": true}` both request the same payload: `telemetry.id + telemetry.summary`.
+
+The object form currently exposes the `summary` switch only.
+
+If you do not want telemetry, either omit the field or set:
+
+```json
+{"telemetry": false}
+```
+
+```json
+{"telemetry": {"summary": false}}
+```
+
+### Multipart upload API
+
+`POST /api/v1/resources/temp_upload` is a multipart form endpoint. For this endpoint, pass telemetry as a form field:
+
+```bash
+curl -X POST http://localhost:1933/api/v1/resources/temp_upload \
+ -H "X-API-Key: your-key" \
+ -F "file=@./notes.md" \
+ -F "telemetry=true"
+```
+
+This endpoint currently accepts the boolean form only.
+
+## Common summary groups
+
+The top-level summary always includes:
+
+- `operation`
+- `status`
+- `duration_ms`
+
+Depending on the operation, you may also see these groups:
+
+- `tokens`: LLM and embedding token totals
+- `vector`: vector search and filtering counts
+- `resource`: resource ingestion and processing stages
+- `queue`: queue processing counts for wait-mode imports
+- `semantic_nodes`: semantic extraction totals
+- `memory`: memory extraction or dedup summaries
+- `errors`: aggregated error information
+
+If a group does not apply to the operation, it is omitted.
+
+## Field reference
+
+Only fields that are actually produced by an operation are returned. Missing groups should be treated as "not applicable" rather than as zero.
+
+### Top-level telemetry fields
+
+| Field | Meaning |
+| --- | --- |
+| `telemetry.id` | Opaque correlation id for this operation |
+| `summary.operation` | Operation name, such as `search.find`, `resources.add_resource`, or `session.commit` |
+| `summary.status` | Final telemetry status, usually `ok` or `error` |
+| `summary.duration_ms` | End-to-end duration of the operation in milliseconds |
+
+### `summary.tokens`
+
+| Field | Meaning |
+| --- | --- |
+| `summary.tokens.total` | Total tokens counted for this operation |
+| `summary.tokens.llm.input` | Total LLM input tokens |
+| `summary.tokens.llm.output` | Total LLM output tokens |
+| `summary.tokens.llm.total` | Total LLM tokens |
+| `summary.tokens.embedding.total` | Total embedding-model tokens |
+
+### `summary.vector`
+
+| Field | Meaning |
+| --- | --- |
+| `summary.vector.searches` | Number of vector search calls |
+| `summary.vector.scored` | Number of candidates that were scored |
+| `summary.vector.passed` | Number of candidates that passed thresholding or later filters |
+| `summary.vector.returned` | Number of results returned to upper-layer logic |
+| `summary.vector.scanned` | Number of vectors scanned by the backend |
+| `summary.vector.scan_reason` | Text description of the scan strategy or reason |
+
+### `summary.resource`
+
+This group appears on resource ingestion operations such as `resources.add_resource`.
+
+| Field | Meaning |
+| --- | --- |
+| `summary.resource.request.duration_ms` | Total request-side duration for the add-resource flow |
+| `summary.resource.process.duration_ms` | Duration of the main resource-processing flow |
+| `summary.resource.process.parse.duration_ms` | Time spent parsing the resource |
+| `summary.resource.process.parse.warnings_count` | Number of parse warnings |
+| `summary.resource.process.finalize.duration_ms` | Time spent finalizing the resource tree |
+| `summary.resource.process.summarize.duration_ms` | Time spent on summarize/vectorize processing |
+| `summary.resource.wait.duration_ms` | Time spent waiting for downstream processing when `wait=true` |
+| `summary.resource.watch.duration_ms` | Time spent creating, updating, or removing watch tasks |
+| `summary.resource.flags.wait` | Whether the request used `wait=true` |
+| `summary.resource.flags.build_index` | Whether the request enabled `build_index` |
+| `summary.resource.flags.summarize` | Whether the request explicitly enabled `summarize` |
+| `summary.resource.flags.watch_enabled` | Whether watch management was enabled for this request |
+
+### `summary.queue`
+
+This group appears when OpenViking waits for queue-backed work to complete.
+
+| Field | Meaning |
+| --- | --- |
+| `summary.queue.semantic.processed` | Number of semantic-queue messages processed |
+| `summary.queue.semantic.error_count` | Number of semantic-queue errors |
+| `summary.queue.embedding.processed` | Number of embedding-queue messages processed |
+| `summary.queue.embedding.error_count` | Number of embedding-queue errors |
+
+### `summary.semantic_nodes`
+
+| Field | Meaning |
+| --- | --- |
+| `summary.semantic_nodes.total` | Total DAG or semantic-node count |
+| `summary.semantic_nodes.done` | Number of completed nodes |
+| `summary.semantic_nodes.pending` | Number of pending nodes |
+| `summary.semantic_nodes.running` | Number of nodes still running |
+
+### `summary.memory`
+
+This group appears on memory-extraction flows such as `session.commit`.
+
+| Field | Meaning |
+| --- | --- |
+| `summary.memory.extracted` | Final number of memories extracted by the operation |
+| `summary.memory.extract.duration_ms` | Total duration of the memory-extraction flow |
+| `summary.memory.extract.candidates.total` | Total extracted candidates before final actions |
+| `summary.memory.extract.candidates.standard` | Standard memory candidates |
+| `summary.memory.extract.candidates.tool_skill` | Tool or skill candidates |
+| `summary.memory.extract.actions.created` | Number of newly created memories |
+| `summary.memory.extract.actions.merged` | Number of merges into existing memories |
+| `summary.memory.extract.actions.deleted` | Number of deleted old memories |
+| `summary.memory.extract.actions.skipped` | Number of skipped candidates |
+| `summary.memory.extract.stages.prepare_inputs_ms` | Time spent preparing extraction inputs |
+| `summary.memory.extract.stages.llm_extract_ms` | Time spent in the LLM extraction call |
+| `summary.memory.extract.stages.normalize_candidates_ms` | Time spent parsing and normalizing candidates |
+| `summary.memory.extract.stages.tool_skill_stats_ms` | Time spent aggregating tool or skill stats |
+| `summary.memory.extract.stages.profile_create_ms` | Time spent creating or updating profile memory |
+| `summary.memory.extract.stages.tool_skill_merge_ms` | Time spent merging tool or skill memories |
+| `summary.memory.extract.stages.dedup_ms` | Time spent deduplicating candidates |
+| `summary.memory.extract.stages.create_memory_ms` | Time spent creating new memories |
+| `summary.memory.extract.stages.merge_existing_ms` | Time spent merging into existing memories |
+| `summary.memory.extract.stages.delete_existing_ms` | Time spent deleting older memories |
+| `summary.memory.extract.stages.create_relations_ms` | Time spent creating used-URI relations |
+| `summary.memory.extract.stages.flush_semantic_ms` | Time spent flushing semantic queue work |
+
+### `summary.errors`
+
+| Field | Meaning |
+| --- | --- |
+| `summary.errors.stage` | Logical stage where the error was recorded |
+| `summary.errors.error_code` | Error code or exception type |
+| `summary.errors.message` | Human-readable error message |
+
+## Examples
+
+### Search request with telemetry
+
+```bash
+curl -X POST http://localhost:1933/api/v1/search/find \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "query": "memory dedup",
+ "limit": 5,
+ "telemetry": true
+ }'
+```
+
+### Add a resource and return telemetry
+
+```bash
+curl -X POST http://localhost:1933/api/v1/resources \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "path": "./docs/readme.md",
+ "reason": "telemetry demo",
+ "wait": true,
+ "telemetry": true
+ }'
+```
+
+### Python SDK
+
+```python
+from openviking import AsyncOpenVikingClient
+
+client = AsyncOpenVikingClient(config_path="/path/to/config.yaml")
+await client.initialize()
+
+result = await client.find("memory dedup", telemetry=True)
+print(result["telemetry"]["summary"]["operation"])
+print(result["telemetry"]["summary"]["duration_ms"])
+```
+
+## Limitations and behavior
+
+- OpenViking currently exposes summary-only telemetry to users
+- `{"telemetry": {"events": true}}` is not a supported public request shape
+- event-stream style selection is not part of the public API
+- `session.commit` supports telemetry only when `wait=true`
+- if you call `session.commit` with `wait=false` and request telemetry, the server returns `INVALID_ARGUMENT`
+- telemetry shape is stable at the top level, but optional summary groups vary by operation
+
+## Related docs
+
+- [Monitoring & Health Checks](05-monitoring.md)
+- [Authentication](04-authentication.md)
+- [System API](../api/07-system.md)
diff --git a/docs/zh/guides/07-operation-telemetry.md b/docs/zh/guides/07-operation-telemetry.md
new file mode 100644
index 000000000..7f74a14b5
--- /dev/null
+++ b/docs/zh/guides/07-operation-telemetry.md
@@ -0,0 +1,302 @@
+# 操作级 Telemetry 使用指南
+
+操作级 telemetry 用来让 OpenViking 在请求结果里额外返回一份结构化摘要,帮助你了解这次操作实际发生了什么,例如耗时、token 消耗、向量检索情况、队列处理进度,以及资源导入阶段统计。
+
+适合这些场景:
+
+- 排查请求为什么变慢
+- 观察 token 或检索行为
+- 把结构化执行摘要接入你自己的日志或观测系统
+
+## 基本说明
+
+Telemetry 是按需返回的。只有你显式请求时,OpenViking 才会在响应顶层返回 `telemetry` 字段。
+
+典型响应结构如下:
+
+```json
+{
+ "status": "ok",
+ "result": {"...": "..."},
+ "telemetry": {
+ "id": "tm_xxx",
+ "summary": {
+ "operation": "search.find",
+ "status": "ok",
+ "duration_ms": 31.2,
+ "tokens": {
+ "total": 24,
+ "llm": {
+ "input": 12,
+ "output": 6,
+ "total": 18
+ }
+ },
+ "vector": {
+ "searches": 3,
+ "scored": 26,
+ "passed": 8,
+ "returned": 5
+ }
+ }
+ }
+}
+```
+
+说明:
+
+- `telemetry.id` 是不透明的关联 ID
+- `telemetry.summary` 是面向调用方的结构化摘要
+- 只有本次操作实际产出的分组才会返回
+- 数值型 `0` 默认不会出现在响应里
+
+## 当前支持范围
+
+### HTTP API
+
+当前这些接口支持 operation telemetry:
+
+- `POST /api/v1/search/find`
+- `POST /api/v1/search/search`
+- `POST /api/v1/resources/temp_upload`
+- `POST /api/v1/resources`
+- `POST /api/v1/skills`
+- `POST /api/v1/sessions/{session_id}/commit`
+
+### Python SDK
+
+Python 客户端里,下面这些调用支持相同的 telemetry 语义:
+
+- `add_resource(...)`
+- `add_skill(...)`
+- `find(...)`
+- `search(...)`
+- `commit_session(...)`
+- `Session.commit(...)`
+
+## 如何请求 telemetry
+
+### JSON 请求
+
+对于 JSON body,`telemetry` 支持下面两种常用写法:
+
+```json
+{"telemetry": true}
+```
+
+```json
+{"telemetry": {"summary": true}}
+```
+
+`true` 和 `{"summary": true}` 的效果相同,都会返回 `telemetry.id + telemetry.summary`。
+
+对象形态当前只开放 `summary` 这个开关。
+
+如果不想返回 telemetry,可以省略该字段,或者显式传:
+
+```json
+{"telemetry": false}
+```
+
+```json
+{"telemetry": {"summary": false}}
+```
+
+### Multipart 上传请求
+
+`POST /api/v1/resources/temp_upload` 是 multipart form 接口。这个接口需要把 telemetry 当作表单字段传入:
+
+```bash
+curl -X POST http://localhost:1933/api/v1/resources/temp_upload \
+ -H "X-API-Key: your-key" \
+ -F "file=@./notes.md" \
+ -F "telemetry=true"
+```
+
+这个接口当前只支持布尔形态的表单参数。
+
+## 常见 summary 分组
+
+summary 顶层这 3 个基础字段总会存在:
+
+- `operation`
+- `status`
+- `duration_ms`
+
+根据不同操作,还可能出现这些分组:
+
+- `tokens`:LLM 和 embedding 的 token 统计
+- `vector`:向量检索与过滤统计
+- `resource`:资源导入与处理阶段摘要
+- `queue`:等待模式下的队列处理统计
+- `semantic_nodes`:语义节点提取统计
+- `memory`:记忆提取或去重摘要
+- `errors`:聚合后的错误信息
+
+如果某个分组对本次操作不适用,就不会返回。
+
+## 字段说明
+
+只有这次操作实际产出的字段才会返回。某个分组缺失时,应理解为“不适用”,而不是默认等于 0。
+
+### 顶层 telemetry 字段
+
+| 字段 | 含义 |
+| --- | --- |
+| `telemetry.id` | 本次操作的不透明关联 ID |
+| `summary.operation` | 操作名,例如 `search.find`、`resources.add_resource`、`session.commit` |
+| `summary.status` | telemetry 最终状态,通常是 `ok` 或 `error` |
+| `summary.duration_ms` | 本次操作的端到端总耗时,单位毫秒 |
+
+### `summary.tokens`
+
+| 字段 | 含义 |
+| --- | --- |
+| `summary.tokens.total` | 本次操作累计 token 总量 |
+| `summary.tokens.llm.input` | LLM 输入 token 总量 |
+| `summary.tokens.llm.output` | LLM 输出 token 总量 |
+| `summary.tokens.llm.total` | LLM token 总量 |
+| `summary.tokens.embedding.total` | embedding 模型 token 总量 |
+
+### `summary.vector`
+
+| 字段 | 含义 |
+| --- | --- |
+| `summary.vector.searches` | 向量检索调用次数 |
+| `summary.vector.scored` | 被打分的候选数量 |
+| `summary.vector.passed` | 通过阈值或后续过滤的候选数量 |
+| `summary.vector.returned` | 最终返回给上层逻辑的结果数量 |
+| `summary.vector.scanned` | 底层实际扫描的向量数量 |
+| `summary.vector.scan_reason` | 本次扫描策略或扫描原因说明 |
+
+### `summary.resource`
+
+这个分组常见于 `resources.add_resource` 这类资源导入操作。
+
+| 字段 | 含义 |
+| --- | --- |
+| `summary.resource.request.duration_ms` | add-resource 请求主流程总耗时 |
+| `summary.resource.process.duration_ms` | 资源处理主流程耗时 |
+| `summary.resource.process.parse.duration_ms` | 资源解析阶段耗时 |
+| `summary.resource.process.parse.warnings_count` | 解析阶段 warning 数量 |
+| `summary.resource.process.finalize.duration_ms` | 资源树 finalize 阶段耗时 |
+| `summary.resource.process.summarize.duration_ms` | summarize 或 vectorize 阶段耗时 |
+| `summary.resource.wait.duration_ms` | `wait=true` 时等待下游处理完成的耗时 |
+| `summary.resource.watch.duration_ms` | 创建、更新或移除 watch 任务的耗时 |
+| `summary.resource.flags.wait` | 本次请求是否使用了 `wait=true` |
+| `summary.resource.flags.build_index` | 本次请求是否启用了 `build_index` |
+| `summary.resource.flags.summarize` | 本次请求是否显式启用了 `summarize` |
+| `summary.resource.flags.watch_enabled` | 本次请求是否启用了 watch 管理 |
+
+### `summary.queue`
+
+这个分组常见于需要等待队列任务完成的操作。
+
+| 字段 | 含义 |
+| --- | --- |
+| `summary.queue.semantic.processed` | 已处理的 semantic queue 消息数 |
+| `summary.queue.semantic.error_count` | semantic queue 错误数 |
+| `summary.queue.embedding.processed` | 已处理的 embedding queue 消息数 |
+| `summary.queue.embedding.error_count` | embedding queue 错误数 |
+
+### `summary.semantic_nodes`
+
+| 字段 | 含义 |
+| --- | --- |
+| `summary.semantic_nodes.total` | DAG 或语义节点总数 |
+| `summary.semantic_nodes.done` | 已完成节点数 |
+| `summary.semantic_nodes.pending` | 待处理节点数 |
+| `summary.semantic_nodes.running` | 正在处理中的节点数 |
+
+### `summary.memory`
+
+这个分组常见于 `session.commit` 这类记忆提取流程。
+
+| 字段 | 含义 |
+| --- | --- |
+| `summary.memory.extracted` | 本次操作最终抽取出的 memory 数量 |
+| `summary.memory.extract.duration_ms` | memory extract 主流程总耗时 |
+| `summary.memory.extract.candidates.total` | 最终动作执行前的候选总数 |
+| `summary.memory.extract.candidates.standard` | 普通 memory candidate 数量 |
+| `summary.memory.extract.candidates.tool_skill` | tool 或 skill candidate 数量 |
+| `summary.memory.extract.actions.created` | 新建 memory 数量 |
+| `summary.memory.extract.actions.merged` | 合并到已有 memory 的次数 |
+| `summary.memory.extract.actions.deleted` | 删除旧 memory 的次数 |
+| `summary.memory.extract.actions.skipped` | 被跳过的 candidate 数量 |
+| `summary.memory.extract.stages.prepare_inputs_ms` | 提取前准备输入数据的耗时 |
+| `summary.memory.extract.stages.llm_extract_ms` | 调用 LLM 做提取的耗时 |
+| `summary.memory.extract.stages.normalize_candidates_ms` | 解析并归一化候选的耗时 |
+| `summary.memory.extract.stages.tool_skill_stats_ms` | 聚合 tool 或 skill 统计的耗时 |
+| `summary.memory.extract.stages.profile_create_ms` | 创建或更新 profile memory 的耗时 |
+| `summary.memory.extract.stages.tool_skill_merge_ms` | 合并 tool 或 skill memory 的耗时 |
+| `summary.memory.extract.stages.dedup_ms` | candidate 去重耗时 |
+| `summary.memory.extract.stages.create_memory_ms` | 创建新 memory 的耗时 |
+| `summary.memory.extract.stages.merge_existing_ms` | 合并到已有 memory 的耗时 |
+| `summary.memory.extract.stages.delete_existing_ms` | 删除旧 memory 的耗时 |
+| `summary.memory.extract.stages.create_relations_ms` | 创建 used-uri relations 的耗时 |
+| `summary.memory.extract.stages.flush_semantic_ms` | flush semantic queue 的耗时 |
+
+### `summary.errors`
+
+| 字段 | 含义 |
+| --- | --- |
+| `summary.errors.stage` | 记录错误时所在的逻辑阶段 |
+| `summary.errors.error_code` | 错误码或异常类型 |
+| `summary.errors.message` | 人类可读的错误描述 |
+
+## 示例
+
+### 带 telemetry 的检索请求
+
+```bash
+curl -X POST http://localhost:1933/api/v1/search/find \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "query": "memory dedup",
+ "limit": 5,
+ "telemetry": true
+ }'
+```
+
+### 导入资源并返回 telemetry
+
+```bash
+curl -X POST http://localhost:1933/api/v1/resources \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "path": "./docs/readme.md",
+ "reason": "telemetry demo",
+ "wait": true,
+ "telemetry": true
+ }'
+```
+
+### Python SDK
+
+```python
+from openviking import AsyncOpenVikingClient
+
+client = AsyncOpenVikingClient(config_path="/path/to/config.yaml")
+await client.initialize()
+
+result = await client.find("memory dedup", telemetry=True)
+print(result["telemetry"]["summary"]["operation"])
+print(result["telemetry"]["summary"]["duration_ms"])
+```
+
+## 限制与注意事项
+
+- 当前对外只提供 summary-only telemetry
+- `{"telemetry": {"events": true}}` 不是当前支持的公开请求形态
+- 事件流风格的选择参数不属于当前公开接口
+- `session.commit` 只有在 `wait=true` 时才支持 telemetry
+- 如果 `session.commit` 使用 `wait=false` 并请求 telemetry,服务端会返回 `INVALID_ARGUMENT`
+- telemetry 的顶层结构稳定,但具体有哪些 summary 分组取决于实际操作
+
+## 相关文档
+
+- [监控与健康检查](05-monitoring.md)
+- [认证](04-authentication.md)
+- [系统 API](../api/07-system.md)
diff --git a/openviking/console/static/app.js b/openviking/console/static/app.js
index 327b28a90..f59656cf0 100644
--- a/openviking/console/static/app.js
+++ b/openviking/console/static/app.js
@@ -1712,6 +1712,7 @@ function bindAddResource() {
const formData = new FormData();
formData.append("file", file);
+ formData.append("telemetry", "true");
setOutput(`Uploading ${file.name} ...`);
const uploadPayload = await callConsole("/ov/resources/temp_upload", {
@@ -1737,6 +1738,10 @@ function bindAddResource() {
upload: uploadPayload.result,
add_resource: addPayload.result,
},
+ telemetry: {
+ upload: uploadPayload.telemetry,
+ add_resource: addPayload.telemetry,
+ },
});
} catch (error) {
setOutput(error.message);
diff --git a/openviking/server/routers/resources.py b/openviking/server/routers/resources.py
index c2fcccf64..94e70a056 100644
--- a/openviking/server/routers/resources.py
+++ b/openviking/server/routers/resources.py
@@ -7,7 +7,7 @@ import uuid
from pathlib import Path
from typing import Any, Optional
-from fastapi import APIRouter, Depends, File, UploadFile
+from fastapi import APIRouter, Depends, File, Form, UploadFile
from pydantic import BaseModel, model_validator
from openviking.server.auth import get_request_context
@@ -109,24 +109,38 @@ def _cleanup_temp_files(temp_dir: Path, max_age_hours: int = 1):
@router.post("/resources/temp_upload")
async def temp_upload(
file: UploadFile = File(...),
+ telemetry: bool = Form(False),
_ctx: RequestContext = Depends(get_request_context),
):
"""Upload a temporary file for add_resource or import_ovpack."""
- config = get_openviking_config()
- temp_dir = config.storage.get_upload_temp_dir()
- # Clean up old temporary files
- _cleanup_temp_files(temp_dir)
+ async def _upload() -> dict[str, str]:
+ config = get_openviking_config()
+ temp_dir = config.storage.get_upload_temp_dir()
- # Save the uploaded file
- file_ext = Path(file.filename).suffix if file.filename else ".tmp"
- temp_filename = f"upload_{uuid.uuid4().hex}{file_ext}"
- temp_file_path = temp_dir / temp_filename
+ # Clean up old temporary files
+ _cleanup_temp_files(temp_dir)
- with open(temp_file_path, "wb") as f:
- f.write(await file.read())
+ # Save the uploaded file
+ file_ext = Path(file.filename).suffix if file.filename else ".tmp"
+ temp_filename = f"upload_{uuid.uuid4().hex}{file_ext}"
+ temp_file_path = temp_dir / temp_filename
- return Response(status="ok", result={"temp_path": str(temp_file_path)})
+ with open(temp_file_path, "wb") as f:
+ f.write(await file.read())
+
+ return {"temp_path": str(temp_file_path)}
+
+ execution = await run_operation(
+ operation="resources.temp_upload",
+ telemetry=telemetry,
+ fn=_upload,
+ )
+ return Response(
+ status="ok",
+ result=execution.result,
+ telemetry=execution.telemetry,
+ ).model_dump(exclude_none=True)
@router.post("/resources")
diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.py
index d41942bcd..5406706d1 100644
--- a/openviking/service/resource_service.py
+++ b/openviking/service/resource_service.py
@@ -147,7 +147,17 @@ class ResourceService:
"""
self._ensure_initialized()
request_start = time.perf_counter()
+ telemetry = get_current_telemetry()
telemetry_id = register_wait_telemetry(wait)
+ watch_manager = self._get_watch_manager()
+ watch_enabled = bool(
+ watch_manager and to and not skip_watch_management and watch_interval > 0
+ )
+
+ telemetry.set("resource.flags.wait", wait)
+ telemetry.set("resource.flags.build_index", build_index)
+ telemetry.set("resource.flags.summarize", summarize)
+ telemetry.set("resource.flags.watch_enabled", watch_enabled)
try:
# add_resource only supports resources scope
@@ -163,7 +173,6 @@ class ResourceService:
raise InvalidArgumentError(
f"add_resource only supports resources scope, use dedicated interface to add {parsed.scope} content"
)
- watch_manager = self._get_watch_manager()
if watch_manager and not skip_watch_management and watch_interval > 0 and not to:
raise InvalidArgumentError(
"watch_interval > 0 requires 'to' to be specified (target URI to watch)"
@@ -186,9 +195,10 @@ class ResourceService:
qm = get_queue_manager()
wait_start = time.perf_counter()
try:
- status = await qm.wait_complete(timeout=timeout)
+ with telemetry.measure("resource.wait"):
+ status = await qm.wait_complete(timeout=timeout)
except TimeoutError as exc:
- get_current_telemetry().set_error(
+ telemetry.set_error(
"resource_service.wait_complete",
"DEADLINE_EXCEEDED",
str(exc),
@@ -201,50 +211,50 @@ class ResourceService:
queue_status=status,
root_uri=result.get("root_uri"),
)
- get_current_telemetry().set("queue.wait.duration_ms", queue_wait_duration_ms)
+ telemetry.set("queue.wait.duration_ms", queue_wait_duration_ms)
if watch_manager and to and not skip_watch_management:
- if watch_interval > 0:
- try:
- processor_kwargs = self._sanitize_watch_processor_kwargs(kwargs)
- await self._handle_watch_task_creation(
- path=path,
- to_uri=to,
- parent_uri=parent,
- reason=reason,
- instruction=instruction,
- watch_interval=watch_interval,
- build_index=build_index,
- summarize=summarize,
- processor_kwargs=processor_kwargs,
- ctx=ctx,
- )
- except ConflictError:
- raise
- except Exception as e:
- logger.warning(
- f"[ResourceService] Failed to create watch task for {to}: {e}"
- )
- else:
- try:
- await self._handle_watch_task_cancellation(to_uri=to, ctx=ctx)
- except Exception as e:
- logger.warning(
- f"[ResourceService] Failed to cancel watch task for {to}: {e}"
- )
-
- get_current_telemetry().set(
- "resource.request.duration_ms",
- round((time.perf_counter() - request_start) * 1000, 3),
- )
+ with telemetry.measure("resource.watch"):
+ if watch_interval > 0:
+ try:
+ processor_kwargs = self._sanitize_watch_processor_kwargs(kwargs)
+ await self._handle_watch_task_creation(
+ path=path,
+ to_uri=to,
+ parent_uri=parent,
+ reason=reason,
+ instruction=instruction,
+ watch_interval=watch_interval,
+ build_index=build_index,
+ summarize=summarize,
+ processor_kwargs=processor_kwargs,
+ ctx=ctx,
+ )
+ except ConflictError:
+ raise
+ except Exception as e:
+ logger.warning(
+ f"[ResourceService] Failed to create watch task for {to}: {e}"
+ )
+ else:
+ try:
+ await self._handle_watch_task_cancellation(to_uri=to, ctx=ctx)
+ except Exception as e:
+ logger.warning(
+ f"[ResourceService] Failed to cancel watch task for {to}: {e}"
+ )
return result
except Exception as exc:
- get_current_telemetry().set_error(
+ telemetry.set_error(
"resource_service.add_resource",
type(exc).__name__,
str(exc),
)
raise
finally:
+ telemetry.set(
+ "resource.request.duration_ms",
+ round((time.perf_counter() - request_start) * 1000, 3),
+ )
unregister_wait_telemetry(telemetry_id)
async def _handle_watch_task_creation(
diff --git a/openviking/session/compressor.py b/openviking/session/compressor.py
index f6c294f97..a55905060 100644
--- a/openviking/session/compressor.py
+++ b/openviking/session/compressor.py
@@ -15,6 +15,7 @@ from openviking.message import Message
from openviking.server.identity import RequestContext
from openviking.storage import VikingDBManager
from openviking.storage.viking_fs import get_viking_fs
+from openviking.telemetry import get_current_telemetry
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils import get_logger
@@ -288,206 +289,254 @@ class SessionCompressor:
return []
self._pending_semantic_changes.clear()
+ telemetry = get_current_telemetry()
+ telemetry.set("memory.extract.candidates.total", 0)
+ telemetry.set("memory.extract.candidates.standard", 0)
+ telemetry.set("memory.extract.candidates.tool_skill", 0)
+ telemetry.set("memory.extract.created", 0)
+ telemetry.set("memory.extract.merged", 0)
+ telemetry.set("memory.extract.deleted", 0)
+ telemetry.set("memory.extract.skipped", 0)
+
+ with telemetry.measure("memory.extract.total"):
+ try:
+ if strict_extract_errors:
+ # Intentionally let extraction errors bubble up so caller (task tracker)
+ # can mark background commit tasks as failed with an explicit error.
+ candidates = await self.extractor.extract_strict(context, user, session_id)
+ else:
+ candidates = await self.extractor.extract(context, user, session_id)
+
+ if not candidates:
+ return []
+
+ tool_skill_count = sum(
+ 1 for candidate in candidates if candidate.category in TOOL_SKILL_CATEGORIES
+ )
+ telemetry.set("memory.extract.candidates.total", len(candidates))
+ telemetry.set("memory.extract.candidates.tool_skill", tool_skill_count)
+ telemetry.set(
+ "memory.extract.candidates.standard",
+ len(candidates) - tool_skill_count,
+ )
- try:
- if strict_extract_errors:
- # Intentionally let extraction errors bubble up so caller (task tracker)
- # can mark background commit tasks as failed with an explicit error.
- candidates = await self.extractor.extract_strict(context, user, session_id)
- else:
- candidates = await self.extractor.extract(context, user, session_id)
-
- if not candidates:
- return []
-
- memories: List[Context] = []
- stats = ExtractionStats()
- # Track created memories' embeddings for batch-internal dedup (#687)
- batch_memories: list[tuple[list[float], Context]] = []
- viking_fs = get_viking_fs()
-
- tool_parts = self._extract_tool_parts(messages)
- from .tool_skill_utils import collect_skill_stats, collect_tool_stats
-
- tool_stats_map = collect_tool_stats(tool_parts)
- skill_stats_map = collect_skill_stats(tool_parts)
-
- for candidate in candidates:
- # Profile: skip dedup, always merge
- if candidate.category in ALWAYS_MERGE_CATEGORIES:
- memory = await self.extractor.create_memory(
- candidate, user, session_id, ctx=ctx
- )
- if memory:
- memories.append(memory)
- stats.created += 1
- await self._index_memory(memory, ctx)
- else:
- stats.skipped += 1
- continue
+ memories: List[Context] = []
+ stats = ExtractionStats()
+ # Track created memories' embeddings for batch-internal dedup (#687)
+ batch_memories: list[tuple[list[float], Context]] = []
+ viking_fs = get_viking_fs()
- # Tool/Skill Memory: 特殊合并逻辑
- if candidate.category in TOOL_SKILL_CATEGORIES:
- if isinstance(candidate, ToolSkillCandidateMemory):
- tool_name, skill_name, tool_status = self._get_tool_skill_info(
- candidate, tool_parts
- )
- candidate.tool_status = tool_status
- if tool_name:
- candidate.tool_name = tool_name
- if skill_name:
- candidate.skill_name = skill_name
-
- if tool_name and candidate.call_time == 0:
- tool_stats = tool_stats_map.get(tool_name, {})
- candidate.call_time = tool_stats.get("call_count", candidate.call_time)
- candidate.success_time = tool_stats.get(
- "success_time", candidate.success_time
- )
- candidate.duration_ms = tool_stats.get(
- "duration_ms", candidate.duration_ms
- )
- candidate.prompt_tokens = tool_stats.get(
- "prompt_tokens", candidate.prompt_tokens
- )
- candidate.completion_tokens = tool_stats.get(
- "completion_tokens", candidate.completion_tokens
- )
+ tool_parts = self._extract_tool_parts(messages)
+ from .tool_skill_utils import collect_skill_stats, collect_tool_stats
- if skill_name and candidate.call_time == 0:
- skill_stats = skill_stats_map.get(skill_name, {})
- candidate.call_time = skill_stats.get("call_count", candidate.call_time)
- candidate.success_time = skill_stats.get(
- "success_time", candidate.success_time
- )
- if skill_name:
- memory = await self.extractor._merge_skill_memory(
- skill_name, candidate, ctx=ctx
- )
- elif tool_name:
- memory = await self.extractor._merge_tool_memory(
- tool_name, candidate, ctx=ctx
+ tool_stats_map = collect_tool_stats(tool_parts)
+ skill_stats_map = collect_skill_stats(tool_parts)
+
+ for candidate in candidates:
+ # Profile: skip dedup, always merge
+ if candidate.category in ALWAYS_MERGE_CATEGORIES:
+ with telemetry.measure("memory.extract.stage.profile_create"):
+ memory = await self.extractor.create_memory(
+ candidate, user, session_id, ctx=ctx
)
- else:
- logger.warning("No tool_name or skill_name found, skipping")
- stats.skipped += 1
- continue
if memory:
memories.append(memory)
- stats.merged += 1
- await self._index_memory(memory, ctx, change_type="modified")
- continue
+ stats.created += 1
+ await self._index_memory(memory, ctx)
+ else:
+ stats.skipped += 1
+ continue
- # Dedup check for other categories
- result = await self.deduplicator.deduplicate(
- candidate, ctx, batch_memories=batch_memories
- )
- actions = result.actions or []
- decision = result.decision
-
- # Safety net: create+merge should be treated as none.
- if decision == DedupDecision.CREATE and any(
- a.decision == MemoryActionDecision.MERGE for a in actions
- ):
- logger.warning(
- f"Dedup returned create with merge action, normalizing to none: "
- f"{candidate.abstract}"
- )
- decision = DedupDecision.NONE
+ # Tool/Skill Memory: 特殊合并逻辑
+ if candidate.category in TOOL_SKILL_CATEGORIES:
+ if isinstance(candidate, ToolSkillCandidateMemory):
+ tool_name, skill_name, tool_status = self._get_tool_skill_info(
+ candidate, tool_parts
+ )
+ candidate.tool_status = tool_status
+ if tool_name:
+ candidate.tool_name = tool_name
+ if skill_name:
+ candidate.skill_name = skill_name
+
+ if tool_name and candidate.call_time == 0:
+ tool_stats = tool_stats_map.get(tool_name, {})
+ candidate.call_time = tool_stats.get(
+ "call_count", candidate.call_time
+ )
+ candidate.success_time = tool_stats.get(
+ "success_time", candidate.success_time
+ )
+ candidate.duration_ms = tool_stats.get(
+ "duration_ms", candidate.duration_ms
+ )
+ candidate.prompt_tokens = tool_stats.get(
+ "prompt_tokens", candidate.prompt_tokens
+ )
+ candidate.completion_tokens = tool_stats.get(
+ "completion_tokens", candidate.completion_tokens
+ )
+
+ if skill_name and candidate.call_time == 0:
+ skill_stats = skill_stats_map.get(skill_name, {})
+ candidate.call_time = skill_stats.get(
+ "call_count", candidate.call_time
+ )
+ candidate.success_time = skill_stats.get(
+ "success_time", candidate.success_time
+ )
+ with telemetry.measure("memory.extract.stage.tool_skill_merge"):
+ if skill_name:
+ memory = await self.extractor._merge_skill_memory(
+ skill_name, candidate, ctx=ctx
+ )
+ elif tool_name:
+ memory = await self.extractor._merge_tool_memory(
+ tool_name, candidate, ctx=ctx
+ )
+ else:
+ memory = None
+ if not tool_name and not skill_name:
+ logger.warning("No tool_name or skill_name found, skipping")
+ stats.skipped += 1
+ continue
+ if memory:
+ memories.append(memory)
+ stats.merged += 1
+ await self._index_memory(memory, ctx, change_type="modified")
+ continue
- if decision == DedupDecision.SKIP:
- stats.skipped += 1
- continue
+ # Dedup check for other categories
+ with telemetry.measure("memory.extract.stage.dedup"):
+ result = await self.deduplicator.deduplicate(
+ candidate, ctx, batch_memories=batch_memories
+ )
+ actions = result.actions or []
+ decision = result.decision
+
+ # Safety net: create+merge should be treated as none.
+ if decision == DedupDecision.CREATE and any(
+ a.decision == MemoryActionDecision.MERGE for a in actions
+ ):
+ logger.warning(
+ f"Dedup returned create with merge action, normalizing to none: "
+ f"{candidate.abstract}"
+ )
+ decision = DedupDecision.NONE
- if decision == DedupDecision.NONE:
- if not actions:
+ if decision == DedupDecision.SKIP:
stats.skipped += 1
continue
- for action in actions:
- if action.decision == MemoryActionDecision.DELETE:
- if viking_fs and await self._delete_existing_memory(
- action.memory, viking_fs, ctx=ctx
- ):
- stats.deleted += 1
- # Remove deleted memory from batch tracking (#687)
- batch_memories = [
- (v, m) for v, m in batch_memories if m.uri != action.memory.uri
- ]
- else:
- stats.skipped += 1
- elif action.decision == MemoryActionDecision.MERGE:
- if candidate.category in MERGE_SUPPORTED_CATEGORIES and viking_fs:
- if await self._merge_into_existing(
- candidate, action.memory, viking_fs, ctx=ctx
- ):
- stats.merged += 1
- # Remove stale batch entry and re-add with updated
- # embedding so 3rd+ candidates can still find it (#687).
+ if decision == DedupDecision.NONE:
+ if not actions:
+ stats.skipped += 1
+ continue
+
+ for action in actions:
+ if action.decision == MemoryActionDecision.DELETE:
+ with telemetry.measure("memory.extract.stage.delete_existing"):
+ deleted = viking_fs and await self._delete_existing_memory(
+ action.memory, viking_fs, ctx=ctx
+ )
+ if deleted:
+ stats.deleted += 1
+ # Remove deleted memory from batch tracking (#687)
batch_memories = [
(v, m)
for v, m in batch_memories
if m.uri != action.memory.uri
]
- if self.deduplicator.embedder:
- merged_text = (
- f"{action.memory.abstract} {candidate.content}"
- )
- merged_embed = self.deduplicator.embedder.embed(merged_text)
- batch_memories.append(
- (merged_embed.dense_vector, action.memory)
+ else:
+ stats.skipped += 1
+ elif action.decision == MemoryActionDecision.MERGE:
+ if candidate.category in MERGE_SUPPORTED_CATEGORIES and viking_fs:
+ with telemetry.measure("memory.extract.stage.merge_existing"):
+ merged = await self._merge_into_existing(
+ candidate, action.memory, viking_fs, ctx=ctx
)
+ if merged:
+ stats.merged += 1
+ # Remove stale batch entry and re-add with updated
+ # embedding so 3rd+ candidates can still find it (#687).
+ batch_memories = [
+ (v, m)
+ for v, m in batch_memories
+ if m.uri != action.memory.uri
+ ]
+ if self.deduplicator.embedder:
+ merged_text = (
+ f"{action.memory.abstract} {candidate.content}"
+ )
+ merged_embed = self.deduplicator.embedder.embed(
+ merged_text
+ )
+ batch_memories.append(
+ (merged_embed.dense_vector, action.memory)
+ )
+ else:
+ stats.skipped += 1
else:
+ # events/cases don't support MERGE, treat as SKIP
stats.skipped += 1
- else:
- # events/cases don't support MERGE, treat as SKIP
- stats.skipped += 1
- continue
-
- if decision == DedupDecision.CREATE:
- # create can optionally include delete actions (delete first, then create)
- for action in actions:
- if action.decision == MemoryActionDecision.DELETE:
- if viking_fs and await self._delete_existing_memory(
- action.memory, viking_fs, ctx=ctx
- ):
- stats.deleted += 1
- # Remove deleted memory from batch tracking (#687)
- batch_memories = [
- (v, m) for v, m in batch_memories if m.uri != action.memory.uri
- ]
- else:
- stats.skipped += 1
+ continue
- memory = await self.extractor.create_memory(
- candidate, user, session_id, ctx=ctx
- )
- if memory:
- memories.append(memory)
- stats.created += 1
- await self._index_memory(memory, ctx)
- # Store embedding for batch-internal dedup of subsequent candidates (#687)
- if result.query_vector:
- batch_memories.append((result.query_vector, memory))
- else:
- stats.skipped += 1
+ if decision == DedupDecision.CREATE:
+ # create can optionally include delete actions (delete first, then create)
+ for action in actions:
+ if action.decision == MemoryActionDecision.DELETE:
+ with telemetry.measure("memory.extract.stage.delete_existing"):
+ deleted = viking_fs and await self._delete_existing_memory(
+ action.memory, viking_fs, ctx=ctx
+ )
+ if deleted:
+ stats.deleted += 1
+ # Remove deleted memory from batch tracking (#687)
+ batch_memories = [
+ (v, m)
+ for v, m in batch_memories
+ if m.uri != action.memory.uri
+ ]
+ else:
+ stats.skipped += 1
- # Extract URIs used in messages, create relations
- used_uris = self._extract_used_uris(messages)
- if used_uris and memories:
- await self._create_relations(memories, used_uris, ctx=ctx)
+ with telemetry.measure("memory.extract.stage.create_memory"):
+ memory = await self.extractor.create_memory(
+ candidate, user, session_id, ctx=ctx
+ )
+ if memory:
+ memories.append(memory)
+ stats.created += 1
+ await self._index_memory(memory, ctx)
+ # Store embedding for batch-internal dedup of subsequent candidates (#687)
+ if result.query_vector:
+ batch_memories.append((result.query_vector, memory))
+ else:
+ stats.skipped += 1
- await self._flush_semantic_operations(ctx)
+ # Extract URIs used in messages, create relations
+ used_uris = self._extract_used_uris(messages)
+ if used_uris and memories:
+ with telemetry.measure("memory.extract.stage.create_relations"):
+ await self._create_relations(memories, used_uris, ctx=ctx)
- logger.info(
- f"Memory extraction: created={stats.created}, "
- f"merged={stats.merged}, deleted={stats.deleted}, skipped={stats.skipped}"
- )
- return memories
+ with telemetry.measure("memory.extract.stage.flush_semantic"):
+ await self._flush_semantic_operations(ctx)
- except Exception:
- self._pending_semantic_changes.clear()
- raise
+ telemetry.set("memory.extract.created", stats.created)
+ telemetry.set("memory.extract.merged", stats.merged)
+ telemetry.set("memory.extract.deleted", stats.deleted)
+ telemetry.set("memory.extract.skipped", stats.skipped)
+
+ logger.info(
+ f"Memory extraction: created={stats.created}, "
+ f"merged={stats.merged}, deleted={stats.deleted}, skipped={stats.skipped}"
+ )
+ return memories
+
+ except Exception:
+ self._pending_semantic_changes.clear()
+ raise
def _extract_tool_parts(self, messages: List[Message]) -> List:
"""Extract all ToolPart from messages."""
diff --git a/openviking/session/memory_extractor.py b/openviking/session/memory_extractor.py
index 99d4a1215..147d8f451 100644
--- a/openviking/session/memory_extractor.py
+++ b/openviking/session/memory_extractor.py
@@ -18,6 +18,7 @@ from openviking.core.context import Context, ContextType, Vectorize
from openviking.prompts import render_prompt
from openviking.server.identity import RequestContext
from openviking.storage.viking_fs import get_viking_fs
+from openviking.telemetry import get_current_telemetry
from openviking_cli.exceptions import NotFoundError
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils import get_logger
@@ -240,6 +241,7 @@ class MemoryExtractor:
logger.warning("LLM not available, skipping memory extraction")
return []
+ telemetry = get_current_telemetry()
messages = context["messages"]
from openviking.message.part import ToolPart
@@ -251,42 +253,50 @@ class MemoryExtractor:
)
tool_parts = []
- for msg in messages:
- for part in getattr(msg, "parts", []):
- if isinstance(part, ToolPart):
- tool_parts.append(part)
-
- tool_stats_map = collect_tool_stats(tool_parts)
- skill_stats_map = collect_skill_stats(tool_parts)
-
- formatted_lines = []
- for m in messages:
- msg_content = self._format_message_with_parts(m)
- if msg_content:
- formatted_lines.append(f"[{m.role}]: {msg_content}")
-
- formatted_messages = "\n".join(formatted_lines)
-
- if not formatted_messages:
- logger.warning("No formatted messages, returning empty list")
- return []
+ tool_stats_map = {}
+ skill_stats_map = {}
+ formatted_messages = ""
+ output_language = "en"
+ prompt = ""
+
+ with telemetry.measure("memory.extract.stage.prepare_inputs"):
+ for msg in messages:
+ for part in getattr(msg, "parts", []):
+ if isinstance(part, ToolPart):
+ tool_parts.append(part)
+
+ formatted_lines = []
+ for m in messages:
+ msg_content = self._format_message_with_parts(m)
+ if msg_content:
+ formatted_lines.append(f"[{m.role}]: {msg_content}")
+
+ formatted_messages = "\n".join(formatted_lines)
+
+ if not formatted_messages:
+ logger.warning("No formatted messages, returning empty list")
+ return []
+
+ config = get_openviking_config()
+ fallback_language = (config.language_fallback or "en").strip() or "en"
+ output_language = self._detect_output_language(
+ messages, fallback_language=fallback_language
+ )
- config = get_openviking_config()
- fallback_language = (config.language_fallback or "en").strip() or "en"
- output_language = self._detect_output_language(
- messages, fallback_language=fallback_language
- )
+ prompt = render_prompt(
+ "compression.memory_extraction",
+ {
+ "summary": "",
+ "recent_messages": formatted_messages,
+ "user": user._user_id,
+ "feedback": "",
+ "output_language": output_language,
+ },
+ )
- prompt = render_prompt(
- "compression.memory_extraction",
- {
- "summary": "",
- "recent_messages": formatted_messages,
- "user": user._user_id,
- "feedback": "",
- "output_language": output_language,
- },
- )
+ with telemetry.measure("memory.extract.stage.tool_skill_stats"):
+ tool_stats_map = collect_tool_stats(tool_parts)
+ skill_stats_map = collect_skill_stats(tool_parts)
try:
from openviking_cli.utils.llm import parse_json_from_response
@@ -298,19 +308,22 @@ class MemoryExtractor:
"recent_messages": formatted_messages,
}
logger.debug("Memory extraction LLM request summary: %s", request_summary)
- response = await vlm.get_completion_async(prompt)
+ with telemetry.measure("memory.extract.stage.llm_extract"):
+ response = await vlm.get_completion_async(prompt)
logger.debug("Memory extraction LLM raw response: %s", response)
- data = parse_json_from_response(response) or {}
- if isinstance(data, list):
- logger.warning(
- "Memory extraction received list instead of dict; wrapping as memories"
- )
- data = {"memories": data}
- elif not isinstance(data, dict):
- logger.warning(
- "Memory extraction received unexpected type %s; skipping", type(data).__name__
- )
- data = {}
+ with telemetry.measure("memory.extract.stage.normalize_candidates"):
+ data = parse_json_from_response(response) or {}
+ if isinstance(data, list):
+ logger.warning(
+ "Memory extraction received list instead of dict; wrapping as memories"
+ )
+ data = {"memories": data}
+ elif not isinstance(data, dict):
+ logger.warning(
+ "Memory extraction received unexpected type %s; skipping",
+ type(data).__name__,
+ )
+ data = {}
logger.debug("Memory extraction LLM parsed payload: %s", data)
candidates = []
@@ -323,80 +336,84 @@ class MemoryExtractor:
# 只在 tools/skills 时使用 ToolSkillCandidateMemory
if category in (MemoryCategory.TOOLS, MemoryCategory.SKILLS):
- llm_tool_name = mem.get("tool_name", "") or ""
- llm_skill_name = mem.get("skill_name", "") or ""
-
- tool_name = ""
- skill_name = ""
- stats = {}
-
- if category == MemoryCategory.TOOLS:
- canonical_tool_name, _ = calibrate_tool_name(llm_tool_name, tool_parts)
- if not canonical_tool_name:
+ with telemetry.measure("memory.extract.stage.tool_skill_stats"):
+ llm_tool_name = mem.get("tool_name", "") or ""
+ llm_skill_name = mem.get("skill_name", "") or ""
+
+ tool_name = ""
+ skill_name = ""
+ stats = {}
+
+ if category == MemoryCategory.TOOLS:
+ canonical_tool_name, _ = calibrate_tool_name(llm_tool_name, tool_parts)
+ if not canonical_tool_name:
+ continue
+ tool_name = canonical_tool_name
+ stats = tool_stats_map.get(tool_name, {})
+
+ if category == MemoryCategory.SKILLS:
+ canonical_skill_name, _ = calibrate_skill_name(
+ llm_skill_name, tool_parts
+ )
+ if not canonical_skill_name:
+ continue
+ skill_name = canonical_skill_name
+ stats = skill_stats_map.get(skill_name, {})
+
+ call_time = stats.get("call_count", 0)
+ if call_time == 0:
continue
- tool_name = canonical_tool_name
- stats = tool_stats_map.get(tool_name, {})
- if category == MemoryCategory.SKILLS:
- canonical_skill_name, _ = calibrate_skill_name(llm_skill_name, tool_parts)
- if not canonical_skill_name:
- continue
- skill_name = canonical_skill_name
- stats = skill_stats_map.get(skill_name, {})
-
- call_time = stats.get("call_count", 0)
- if call_time == 0:
- continue
-
- candidates.append(
- ToolSkillCandidateMemory(
- category=category,
- abstract=mem.get("abstract", ""),
- overview=mem.get("overview", ""),
- content=mem.get("content", ""),
- source_session=session_id,
- user=user,
- language=output_language,
- tool_name=tool_name,
- skill_name=skill_name,
- call_time=call_time,
- success_time=stats.get("success_time", 0),
- duration_ms=(
- stats.get("duration_ms", 0)
- if category == MemoryCategory.TOOLS
- else 0
- ),
- prompt_tokens=(
- stats.get("prompt_tokens", 0)
- if category == MemoryCategory.TOOLS
- else 0
- ),
- completion_tokens=(
- stats.get("completion_tokens", 0)
- if category == MemoryCategory.TOOLS
- else 0
- ),
- best_for=str(mem.get("best_for", "") or "").strip(),
- optimal_params=str(mem.get("optimal_params", "") or "").strip(),
- recommended_flow=str(mem.get("recommended_flow", "") or "").strip(),
- key_dependencies=str(mem.get("key_dependencies", "") or "").strip(),
- common_failures=str(mem.get("common_failures", "") or "").strip(),
- recommendation=str(mem.get("recommendation", "") or "").strip(),
+ candidates.append(
+ ToolSkillCandidateMemory(
+ category=category,
+ abstract=mem.get("abstract", ""),
+ overview=mem.get("overview", ""),
+ content=mem.get("content", ""),
+ source_session=session_id,
+ user=user,
+ language=output_language,
+ tool_name=tool_name,
+ skill_name=skill_name,
+ call_time=call_time,
+ success_time=stats.get("success_time", 0),
+ duration_ms=(
+ stats.get("duration_ms", 0)
+ if category == MemoryCategory.TOOLS
+ else 0
+ ),
+ prompt_tokens=(
+ stats.get("prompt_tokens", 0)
+ if category == MemoryCategory.TOOLS
+ else 0
+ ),
+ completion_tokens=(
+ stats.get("completion_tokens", 0)
+ if category == MemoryCategory.TOOLS
+ else 0
+ ),
+ best_for=str(mem.get("best_for", "") or "").strip(),
+ optimal_params=str(mem.get("optimal_params", "") or "").strip(),
+ recommended_flow=str(mem.get("recommended_flow", "") or "").strip(),
+ key_dependencies=str(mem.get("key_dependencies", "") or "").strip(),
+ common_failures=str(mem.get("common_failures", "") or "").strip(),
+ recommendation=str(mem.get("recommendation", "") or "").strip(),
+ )
)
- )
else:
# 现有逻辑不变,前向兼容
- candidates.append(
- CandidateMemory(
- category=category,
- abstract=mem.get("abstract", ""),
- overview=mem.get("overview", ""),
- content=mem.get("content", ""),
- source_session=session_id,
- user=user,
- language=output_language,
+ with telemetry.measure("memory.extract.stage.normalize_candidates"):
+ candidates.append(
+ CandidateMemory(
+ category=category,
+ abstract=mem.get("abstract", ""),
+ overview=mem.get("overview", ""),
+ content=mem.get("content", ""),
+ source_session=session_id,
+ user=user,
+ language=output_language,
+ )
)
- )
logger.info(
f"Extracted {len(candidates)} candidate memories (language={output_language})"
diff --git a/openviking/telemetry/operation.py b/openviking/telemetry/operation.py
index 82a204918..fc5777d39 100644
--- a/openviking/telemetry/operation.py
+++ b/openviking/telemetry/operation.py
@@ -6,9 +6,10 @@ from __future__ import annotations
import time
from collections import defaultdict
+from contextlib import contextmanager
from dataclasses import dataclass
from threading import Lock
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Iterator, Optional
from uuid import uuid4
@@ -39,6 +40,29 @@ class TelemetrySnapshot:
class TelemetrySummaryBuilder:
"""Build normalized summary metrics from collector data."""
+ _PRUNED = object()
+
+ _MEMORY_EXTRACT_STAGE_KEYS = {
+ "prepare_inputs_ms": "memory.extract.stage.prepare_inputs.duration_ms",
+ "llm_extract_ms": "memory.extract.stage.llm_extract.duration_ms",
+ "normalize_candidates_ms": "memory.extract.stage.normalize_candidates.duration_ms",
+ "tool_skill_stats_ms": "memory.extract.stage.tool_skill_stats.duration_ms",
+ "profile_create_ms": "memory.extract.stage.profile_create.duration_ms",
+ "tool_skill_merge_ms": "memory.extract.stage.tool_skill_merge.duration_ms",
+ "dedup_ms": "memory.extract.stage.dedup.duration_ms",
+ "create_memory_ms": "memory.extract.stage.create_memory.duration_ms",
+ "merge_existing_ms": "memory.extract.stage.merge_existing.duration_ms",
+ "delete_existing_ms": "memory.extract.stage.delete_existing.duration_ms",
+ "create_relations_ms": "memory.extract.stage.create_relations.duration_ms",
+ "flush_semantic_ms": "memory.extract.stage.flush_semantic.duration_ms",
+ }
+ _RESOURCE_FLAG_KEYS = {
+ "wait": "resource.flags.wait",
+ "build_index": "resource.flags.build_index",
+ "summarize": "resource.flags.summarize",
+ "watch_enabled": "resource.flags.watch_enabled",
+ }
+
@staticmethod
def _i(value: Any, default: int = 0) -> int:
if value is None:
@@ -48,6 +72,50 @@ class TelemetrySummaryBuilder:
except (TypeError, ValueError):
return default
+ @staticmethod
+ def _f(value: Any, default: float = 0.0) -> float:
+ if value is None:
+ return default
+ try:
+ return round(float(value), 3)
+ except (TypeError, ValueError):
+ return default
+
+ @staticmethod
+ def _bool(value: Any, default: bool = False) -> bool:
+ if value is None:
+ return default
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, (int, float)):
+ return bool(value)
+ if isinstance(value, str):
+ normalized = value.strip().lower()
+ if normalized in {"true", "1", "yes", "on"}:
+ return True
+ if normalized in {"false", "0", "no", "off", ""}:
+ return False
+ return default
+
+ @classmethod
+ def _prune_zero_metrics(cls, value: Any) -> Any:
+ if isinstance(value, dict):
+ pruned: Dict[str, Any] = {}
+ for key, child in value.items():
+ pruned_child = cls._prune_zero_metrics(child)
+ if pruned_child is cls._PRUNED:
+ continue
+ pruned[key] = pruned_child
+ return pruned if pruned else cls._PRUNED
+
+ if isinstance(value, bool):
+ return value
+
+ if isinstance(value, (int, float)) and value == 0:
+ return cls._PRUNED
+
+ return value
+
@classmethod
def _has_metric_prefix(
cls, prefix: str, counters: Dict[str, float], gauges: Dict[str, Any]
@@ -130,9 +198,59 @@ class TelemetrySummaryBuilder:
}
if cls._has_metric_prefix("memory", counters, gauges):
- summary["memory"] = {
+ memory_summary = {
"extracted": memories_extracted,
}
+ if cls._has_metric_prefix("memory.extract", counters, gauges):
+ memory_summary["extract"] = {
+ "duration_ms": cls._f(gauges.get("memory.extract.total.duration_ms"), 0.0),
+ "candidates": {
+ "total": cls._i(gauges.get("memory.extract.candidates.total"), 0),
+ "standard": cls._i(gauges.get("memory.extract.candidates.standard"), 0),
+ "tool_skill": cls._i(gauges.get("memory.extract.candidates.tool_skill"), 0),
+ },
+ "actions": {
+ "created": cls._i(gauges.get("memory.extract.created"), 0),
+ "merged": cls._i(gauges.get("memory.extract.merged"), 0),
+ "deleted": cls._i(gauges.get("memory.extract.deleted"), 0),
+ "skipped": cls._i(gauges.get("memory.extract.skipped"), 0),
+ },
+ "stages": {
+ public_key: cls._f(gauges.get(metric_key), 0.0)
+ for public_key, metric_key in cls._MEMORY_EXTRACT_STAGE_KEYS.items()
+ },
+ }
+ summary["memory"] = memory_summary
+
+ if cls._has_metric_prefix("resource", counters, gauges):
+ summary["resource"] = {
+ "request": {
+ "duration_ms": cls._f(gauges.get("resource.request.duration_ms"), 0.0),
+ },
+ "process": {
+ "duration_ms": cls._f(gauges.get("resource.process.duration_ms"), 0.0),
+ "parse": {
+ "duration_ms": cls._f(gauges.get("resource.parse.duration_ms"), 0.0),
+ "warnings_count": cls._i(gauges.get("resource.parse.warnings_count"), 0),
+ },
+ "finalize": {
+ "duration_ms": cls._f(gauges.get("resource.finalize.duration_ms"), 0.0),
+ },
+ "summarize": {
+ "duration_ms": cls._f(gauges.get("resource.summarize.duration_ms"), 0.0),
+ },
+ },
+ "wait": {
+ "duration_ms": cls._f(gauges.get("resource.wait.duration_ms"), 0.0),
+ },
+ "watch": {
+ "duration_ms": cls._f(gauges.get("resource.watch.duration_ms"), 0.0),
+ },
+ "flags": {
+ public_key: cls._bool(gauges.get(metric_key), False)
+ for public_key, metric_key in cls._RESOURCE_FLAG_KEYS.items()
+ },
+ }
if error_stage or error_code or error_message:
summary["errors"] = {
@@ -141,6 +259,15 @@ class TelemetrySummaryBuilder:
"message": error_message,
}
+ for key in ("tokens", "queue", "vector", "semantic_nodes", "memory", "resource", "errors"):
+ if key not in summary:
+ continue
+ pruned_value = cls._prune_zero_metrics(summary[key])
+ if pruned_value is cls._PRUNED:
+ summary.pop(key, None)
+ else:
+ summary[key] = pruned_value
+
return summary
@@ -181,6 +308,34 @@ class OperationTelemetry:
def set_value(self, key: str, value: Any) -> None:
self.set(key, value)
+ def add_duration(self, key: str, duration_ms: float) -> None:
+ if not self.enabled:
+ return
+ gauge_key = key if key.endswith(".duration_ms") else f"{key}.duration_ms"
+ try:
+ normalized_duration = max(float(duration_ms), 0.0)
+ except (TypeError, ValueError):
+ normalized_duration = 0.0
+ with self._lock:
+ existing = self._gauges.get(gauge_key, 0.0)
+ try:
+ existing_value = float(existing)
+ except (TypeError, ValueError):
+ existing_value = 0.0
+ self._gauges[gauge_key] = existing_value + normalized_duration
+
+ @contextmanager
+ def measure(self, key: str) -> Iterator[None]:
+ if not self.enabled:
+ yield
+ return
+
+ start = time.perf_counter()
+ try:
+ yield
+ finally:
+ self.add_duration(key, (time.perf_counter() - start) * 1000)
+
def add_token_usage(self, input_tokens: int, output_tokens: int) -> None:
self.add_token_usage_by_source("llm", input_tokens, output_tokens)
diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py
index 172fcd332..ef6185dc3 100644
--- a/openviking/utils/resource_processor.py
+++ b/openviking/utils/resource_processor.py
@@ -124,167 +124,169 @@ class ResourceProcessor:
}
telemetry = get_current_telemetry()
- # ============ Phase 1: Parse source and writes to temp viking fs ============
- try:
- parse_start = time.perf_counter()
- media_processor = self._get_media_processor()
- viking_fs = get_viking_fs()
- # Use reason as instruction fallback so it influences L0/L1
- # generation and improves search relevance as documented.
- effective_instruction = instruction or reason
- with viking_fs.bind_request_context(ctx):
- parse_result = await media_processor.process(
- source=path,
- instruction=effective_instruction,
- **kwargs,
- )
- result["source_path"] = parse_result.source_path or path
- result["meta"] = parse_result.meta
-
- # Only abort when no temp content was produced at all.
- # For directory imports partial success (some files failed) is
- # normal – finalization should still proceed.
- if not parse_result.temp_dir_path:
- result["status"] = "error"
- result["errors"].extend(
- parse_result.warnings or ["Parse failed: no content generated"],
- )
- return result
-
- if parse_result.warnings:
- result["errors"].extend(parse_result.warnings)
-
- telemetry.set(
- "resource.parse.duration_ms",
- round((time.perf_counter() - parse_start) * 1000, 3),
- )
- telemetry.set("resource.parse.warnings_count", len(parse_result.warnings or []))
-
- except Exception as e:
- result["status"] = "error"
- result["errors"].append(f"Parse error: {e}")
- logger.error(f"[ResourceProcessor] Parse error: {e}")
- telemetry.set_error("resource_processor.parse", "PROCESSING_ERROR", str(e))
- import traceback
-
- traceback.print_exc()
- return result
-
- # parse_result contains:
- # - root: ResourceNode tree (with L0/L1 in meta)
- # - temp_dir_path: Temporary directory path (Parser wrote all files)
- # - source_path, source_format
-
- # ============ Phase 3: TreeBuilder finalizes from temp (scan + move to AGFS) ============
- try:
- finalize_start = time.perf_counter()
- with get_viking_fs().bind_request_context(ctx):
- context_tree = await self.tree_builder.finalize_from_temp(
- temp_dir_path=parse_result.temp_dir_path,
- ctx=ctx,
- scope=scope,
- to_uri=to,
- parent_uri=parent,
- source_path=parse_result.source_path,
- source_format=parse_result.source_format,
- )
- if context_tree and context_tree.root:
- result["root_uri"] = context_tree.root.uri
- result["temp_uri"] = context_tree.root.temp_uri
- telemetry.set(
- "resource.finalize.duration_ms",
- round((time.perf_counter() - finalize_start) * 1000, 3),
- )
- except Exception as e:
- result["status"] = "error"
- result["errors"].append(f"Finalize from temp error: {e}")
- telemetry.set_error("resource_processor.finalize", "PROCESSING_ERROR", str(e))
-
- # Cleanup temporary directory on error (via VikingFS)
+ with telemetry.measure("resource.process"):
+ # ============ Phase 1: Parse source and writes to temp viking fs ============
try:
- if parse_result.temp_dir_path:
- await get_viking_fs().delete_temp(parse_result.temp_dir_path, ctx=ctx)
- except Exception:
- pass
-
- return result
-
- # ============ Phase 3.5: 首次添加立即落盘 + 生命周期锁 ============
- root_uri = result.get("root_uri")
- temp_uri = result.get("temp_uri") # temp_doc_uri
- candidate_uri = getattr(context_tree, "_candidate_uri", None) if context_tree else None
- lifecycle_lock_handle_id = ""
-
- if root_uri and temp_uri:
- from openviking.storage.transaction import LockContext, get_lock_manager
+ parse_start = time.perf_counter()
+ media_processor = self._get_media_processor()
+ viking_fs = get_viking_fs()
+ # Use reason as instruction fallback so it influences L0/L1
+ # generation and improves search relevance as documented.
+ effective_instruction = instruction or reason
+ with viking_fs.bind_request_context(ctx):
+ parse_result = await media_processor.process(
+ source=path,
+ instruction=effective_instruction,
+ **kwargs,
+ )
+ result["source_path"] = parse_result.source_path or path
+ result["meta"] = parse_result.meta
+
+ # Only abort when no temp content was produced at all.
+ # For directory imports partial success (some files failed) is
+ # normal - finalization should still proceed.
+ if not parse_result.temp_dir_path:
+ result["status"] = "error"
+ result["errors"].extend(
+ parse_result.warnings or ["Parse failed: no content generated"],
+ )
+ return result
- viking_fs = get_viking_fs()
- lock_manager = get_lock_manager()
- target_exists = await viking_fs.exists(root_uri, ctx=ctx)
+ if parse_result.warnings:
+ result["errors"].extend(parse_result.warnings)
- if not target_exists:
- # 第一次添加:锁保护下将 temp 移到 final
- dst_path = viking_fs._uri_to_path(root_uri, ctx=ctx)
- parent_path = dst_path.rsplit("/", 1)[0] if "/" in dst_path else dst_path
+ telemetry.set(
+ "resource.parse.duration_ms",
+ round((time.perf_counter() - parse_start) * 1000, 3),
+ )
+ telemetry.set("resource.parse.warnings_count", len(parse_result.warnings or []))
- parent_uri = "/".join(root_uri.rsplit("/", 1)[:-1])
- if parent_uri:
- await viking_fs.mkdir(parent_uri, exist_ok=True, ctx=ctx)
+ except Exception as e:
+ result["status"] = "error"
+ result["errors"].append(f"Parse error: {e}")
+ logger.error(f"[ResourceProcessor] Parse error: {e}")
+ telemetry.set_error("resource_processor.parse", "PROCESSING_ERROR", str(e))
+ import traceback
- async with LockContext(lock_manager, [parent_path], lock_mode="point"):
- if candidate_uri:
- root_uri = await self.tree_builder._resolve_unique_uri(candidate_uri)
- result["root_uri"] = root_uri
- dst_path = viking_fs._uri_to_path(root_uri, ctx=ctx)
+ traceback.print_exc()
+ return result
- src_path = viking_fs._uri_to_path(temp_uri, ctx=ctx)
- await asyncio.to_thread(viking_fs.agfs.mv, src_path, dst_path)
+ # parse_result contains:
+ # - root: ResourceNode tree (with L0/L1 in meta)
+ # - temp_dir_path: Temporary directory path (Parser wrote all files)
+ # - source_path, source_format
- # 在 POINT 锁内获取 SUBTREE 锁(消除竞态窗口)
- lifecycle_lock_handle_id = await self._try_acquire_lifecycle_lock(
- lock_manager, dst_path
+ # ============ Phase 3: TreeBuilder finalizes from temp (scan + move to AGFS) ============
+ try:
+ finalize_start = time.perf_counter()
+ with get_viking_fs().bind_request_context(ctx):
+ context_tree = await self.tree_builder.finalize_from_temp(
+ temp_dir_path=parse_result.temp_dir_path,
+ ctx=ctx,
+ scope=scope,
+ to_uri=to,
+ parent_uri=parent,
+ source_path=parse_result.source_path,
+ source_format=parse_result.source_format,
)
+ if context_tree and context_tree.root:
+ result["root_uri"] = context_tree.root.uri
+ result["temp_uri"] = context_tree.root.temp_uri
+ telemetry.set(
+ "resource.finalize.duration_ms",
+ round((time.perf_counter() - finalize_start) * 1000, 3),
+ )
+ except Exception as e:
+ result["status"] = "error"
+ result["errors"].append(f"Finalize from temp error: {e}")
+ telemetry.set_error("resource_processor.finalize", "PROCESSING_ERROR", str(e))
+ # Cleanup temporary directory on error (via VikingFS)
try:
- await viking_fs.delete_temp(parse_result.temp_dir_path, ctx=ctx)
+ if parse_result.temp_dir_path:
+ await get_viking_fs().delete_temp(parse_result.temp_dir_path, ctx=ctx)
except Exception:
pass
- result["temp_uri"] = root_uri
- else:
- # 增量更新:对目标目录加 SUBTREE 锁
- resource_path = viking_fs._uri_to_path(root_uri, ctx=ctx)
- lifecycle_lock_handle_id = await self._try_acquire_lifecycle_lock(
- lock_manager, resource_path
- )
+ return result
- # ============ Phase 4: Optional Steps ============
- build_index = kwargs.get("build_index", True)
- temp_uri_for_summarize = result.get("temp_uri") or parse_result.temp_dir_path
- should_summarize = summarize or build_index
- if should_summarize:
- skip_vec = not build_index
- try:
- await self._get_summarizer().summarize(
- resource_uris=[result["root_uri"]],
- ctx=ctx,
- skip_vectorization=skip_vec,
- lifecycle_lock_handle_id=lifecycle_lock_handle_id,
- temp_uris=[temp_uri_for_summarize],
- **kwargs,
- )
- except Exception as e:
- logger.error(f"Summarization failed: {e}")
- result["warnings"] = result.get("warnings", []) + [f"Summarization failed: {e}"]
- elif lifecycle_lock_handle_id:
- # 无下游处理接管锁,主动释放
- from openviking.storage.transaction import get_lock_manager
+ # ============ Phase 3.5: 首次添加立即落盘 + 生命周期锁 ============
+ root_uri = result.get("root_uri")
+ temp_uri = result.get("temp_uri") # temp_doc_uri
+ candidate_uri = getattr(context_tree, "_candidate_uri", None) if context_tree else None
+ lifecycle_lock_handle_id = ""
+
+ if root_uri and temp_uri:
+ from openviking.storage.transaction import LockContext, get_lock_manager
+
+ viking_fs = get_viking_fs()
+ lock_manager = get_lock_manager()
+ target_exists = await viking_fs.exists(root_uri, ctx=ctx)
+
+ if not target_exists:
+ # 第一次添加:锁保护下将 temp 移到 final
+ dst_path = viking_fs._uri_to_path(root_uri, ctx=ctx)
+ parent_path = dst_path.rsplit("/", 1)[0] if "/" in dst_path else dst_path
+
+ parent_uri = "/".join(root_uri.rsplit("/", 1)[:-1])
+ if parent_uri:
+ await viking_fs.mkdir(parent_uri, exist_ok=True, ctx=ctx)
+
+ async with LockContext(lock_manager, [parent_path], lock_mode="point"):
+ if candidate_uri:
+ root_uri = await self.tree_builder._resolve_unique_uri(candidate_uri)
+ result["root_uri"] = root_uri
+ dst_path = viking_fs._uri_to_path(root_uri, ctx=ctx)
+
+ src_path = viking_fs._uri_to_path(temp_uri, ctx=ctx)
+ await asyncio.to_thread(viking_fs.agfs.mv, src_path, dst_path)
+
+ # 在 POINT 锁内获取 SUBTREE 锁(消除竞态窗口)
+ lifecycle_lock_handle_id = await self._try_acquire_lifecycle_lock(
+ lock_manager, dst_path
+ )
+
+ try:
+ await viking_fs.delete_temp(parse_result.temp_dir_path, ctx=ctx)
+ except Exception:
+ pass
+
+ result["temp_uri"] = root_uri
+ else:
+ # 增量更新:对目标目录加 SUBTREE 锁
+ resource_path = viking_fs._uri_to_path(root_uri, ctx=ctx)
+ lifecycle_lock_handle_id = await self._try_acquire_lifecycle_lock(
+ lock_manager, resource_path
+ )
- handle = get_lock_manager().get_handle(lifecycle_lock_handle_id)
- if handle:
- await get_lock_manager().release(handle)
+ # ============ Phase 4: Optional Steps ============
+ build_index = kwargs.get("build_index", True)
+ temp_uri_for_summarize = result.get("temp_uri") or parse_result.temp_dir_path
+ should_summarize = summarize or build_index
+ if should_summarize:
+ skip_vec = not build_index
+ try:
+ with telemetry.measure("resource.summarize"):
+ await self._get_summarizer().summarize(
+ resource_uris=[result["root_uri"]],
+ ctx=ctx,
+ skip_vectorization=skip_vec,
+ lifecycle_lock_handle_id=lifecycle_lock_handle_id,
+ temp_uris=[temp_uri_for_summarize],
+ **kwargs,
+ )
+ except Exception as e:
+ logger.error(f"Summarization failed: {e}")
+ result["warnings"] = result.get("warnings", []) + [f"Summarization failed: {e}"]
+ elif lifecycle_lock_handle_id:
+ # 无下游处理接管锁,主动释放
+ from openviking.storage.transaction import get_lock_manager
+
+ handle = get_lock_manager().get_handle(lifecycle_lock_handle_id)
+ if handle:
+ await get_lock_manager().release(handle)
- return result
+ return result
@staticmethod
async def _try_acquire_lifecycle_lock(lock_manager, path: str) -> str:
diff --git a/tests/server/test_api_resources.py b/tests/server/test_api_resources.py
index b6d96785b..2159ac6ac 100644
--- a/tests/server/test_api_resources.py
+++ b/tests/server/test_api_resources.py
@@ -3,8 +3,12 @@
"""Tests for resource management endpoints."""
+from types import SimpleNamespace
+
import httpx
+from openviking.telemetry import get_current_telemetry
+
async def test_add_resource_success(client: httpx.AsyncClient, sample_markdown_file):
resp = await client.post(
@@ -56,13 +60,62 @@ async def test_add_resource_with_telemetry_wait(client: httpx.AsyncClient, sampl
telemetry_summary = body["telemetry"]["summary"]
assert telemetry_summary["operation"] == "resources.add_resource"
assert "usage" not in body
- semantic = telemetry_summary["semantic_nodes"]
- assert semantic["total"] is None or semantic["done"] == semantic["total"]
- assert semantic["pending"] in (None, 0)
- assert semantic["running"] in (None, 0)
+ semantic = telemetry_summary.get("semantic_nodes")
+ if semantic is not None:
+ assert semantic["total"] is None or semantic["done"] == semantic["total"]
+ assert semantic["pending"] in (None, 0)
+ assert semantic["running"] in (None, 0)
+ assert "resource" in telemetry_summary
assert "memory" not in telemetry_summary
+async def test_add_resource_with_telemetry_includes_resource_breakdown(
+ client: httpx.AsyncClient, service, monkeypatch
+):
+ async def fake_add_resource(**kwargs):
+ telemetry = get_current_telemetry()
+ telemetry.set("resource.request.duration_ms", 152.3)
+ telemetry.set("resource.process.duration_ms", 101.7)
+ telemetry.set("resource.parse.duration_ms", 38.1)
+ telemetry.set("resource.parse.warnings_count", 1)
+ telemetry.set("resource.finalize.duration_ms", 22.4)
+ telemetry.set("resource.summarize.duration_ms", 31.8)
+ telemetry.set("resource.wait.duration_ms", 46.9)
+ telemetry.set("resource.watch.duration_ms", 0.8)
+ telemetry.set("resource.flags.wait", True)
+ telemetry.set("resource.flags.build_index", True)
+ telemetry.set("resource.flags.summarize", False)
+ telemetry.set("resource.flags.watch_enabled", False)
+ return {
+ "status": "success",
+ "root_uri": "viking://resources/demo",
+ }
+
+ monkeypatch.setattr(service.resources, "add_resource", fake_add_resource)
+
+ resp = await client.post(
+ "/api/v1/resources",
+ json={
+ "path": "/tmp/demo.md",
+ "reason": "telemetry resource",
+ "wait": True,
+ "telemetry": True,
+ },
+ )
+ assert resp.status_code == 200
+ body = resp.json()
+ resource = body["telemetry"]["summary"]["resource"]
+ assert resource["request"]["duration_ms"] == 152.3
+ assert resource["process"]["parse"] == {"duration_ms": 38.1, "warnings_count": 1}
+ assert resource["wait"]["duration_ms"] == 46.9
+ assert resource["flags"] == {
+ "wait": True,
+ "build_index": True,
+ "summarize": False,
+ "watch_enabled": False,
+ }
+
+
async def test_add_resource_with_summary_only_telemetry(
client: httpx.AsyncClient, sample_markdown_file
):
@@ -153,7 +206,9 @@ async def test_wait_processed_after_add(client: httpx.AsyncClient, sample_markdo
assert resp.json()["status"] == "ok"
-async def test_add_resource_with_watch_interval(client: httpx.AsyncClient, sample_markdown_file):
+async def test_add_resource_with_watch_interval_requires_to(
+ client: httpx.AsyncClient, sample_markdown_file
+):
resp = await client.post(
"/api/v1/resources",
json={
@@ -162,10 +217,10 @@ async def test_add_resource_with_watch_interval(client: httpx.AsyncClient, sampl
"watch_interval": 5.0,
},
)
- assert resp.status_code == 200
+ assert resp.status_code == 400
body = resp.json()
- assert body["status"] == "ok"
- assert "root_uri" in body["result"]
+ assert body["status"] == "error"
+ assert "watch_interval > 0 requires 'to' to be specified" in body["error"]["message"]
async def test_add_resource_with_default_watch_interval(
@@ -182,3 +237,44 @@ async def test_add_resource_with_default_watch_interval(
body = resp.json()
assert body["status"] == "ok"
assert "root_uri" in body["result"]
+
+
+async def test_temp_upload_success(client: httpx.AsyncClient, temp_dir, monkeypatch):
+ monkeypatch.setattr(
+ "openviking.server.routers.resources.get_openviking_config",
+ lambda: SimpleNamespace(
+ storage=SimpleNamespace(get_upload_temp_dir=lambda: temp_dir),
+ ),
+ )
+
+ resp = await client.post(
+ "/api/v1/resources/temp_upload",
+ files={"file": ("sample.md", b"# upload\n", "text/markdown")},
+ )
+ assert resp.status_code == 200
+ body = resp.json()
+ assert body["status"] == "ok"
+ assert "telemetry" not in body
+ assert body["result"]["temp_path"].endswith(".md")
+
+
+async def test_temp_upload_with_telemetry_returns_summary(
+ client: httpx.AsyncClient, temp_dir, monkeypatch
+):
+ monkeypatch.setattr(
+ "openviking.server.routers.resources.get_openviking_config",
+ lambda: SimpleNamespace(
+ storage=SimpleNamespace(get_upload_temp_dir=lambda: temp_dir),
+ ),
+ )
+
+ resp = await client.post(
+ "/api/v1/resources/temp_upload",
+ files={"file": ("sample.md", b"# upload\n", "text/markdown")},
+ data={"telemetry": "true"},
+ )
+ assert resp.status_code == 200
+ body = resp.json()
+ assert body["status"] == "ok"
+ assert body["result"]["temp_path"].endswith(".md")
+ assert body["telemetry"]["summary"]["operation"] == "resources.temp_upload"
diff --git a/tests/server/test_api_sessions.py b/tests/server/test_api_sessions.py
index 370e71cf8..368db431f 100644
--- a/tests/server/test_api_sessions.py
+++ b/tests/server/test_api_sessions.py
@@ -6,6 +6,7 @@
import httpx
from openviking.server.identity import RequestContext, Role
+from openviking.telemetry import get_current_telemetry
from openviking_cli.session.user_id import UserIdentifier
@@ -167,10 +168,65 @@ async def test_compress_session_with_telemetry(client: httpx.AsyncClient):
assert summary["operation"] == "session.commit"
assert {"total", "llm", "embedding"}.issubset(summary["tokens"].keys())
assert summary["memory"]["extracted"] is not None
+ assert "extract" in summary["memory"]
assert "semantic_nodes" not in summary
assert "usage" not in body
+async def test_compress_session_with_telemetry_includes_memory_extract_breakdown(
+ client: httpx.AsyncClient, service, monkeypatch
+):
+ create_resp = await client.post("/api/v1/sessions", json={})
+ session_id = create_resp.json()["result"]["session_id"]
+
+ async def fake_commit_async(_session_id: str, _ctx):
+ telemetry = get_current_telemetry()
+ telemetry.set("memory.extracted", 2)
+ telemetry.set("memory.extract.total.duration_ms", 321.5)
+ telemetry.set("memory.extract.candidates.total", 4)
+ telemetry.set("memory.extract.candidates.standard", 3)
+ telemetry.set("memory.extract.candidates.tool_skill", 1)
+ telemetry.set("memory.extract.created", 1)
+ telemetry.set("memory.extract.merged", 1)
+ telemetry.set("memory.extract.deleted", 0)
+ telemetry.set("memory.extract.skipped", 2)
+ telemetry.set("memory.extract.stage.prepare_inputs.duration_ms", 5.0)
+ telemetry.set("memory.extract.stage.llm_extract.duration_ms", 200.0)
+ telemetry.set("memory.extract.stage.normalize_candidates.duration_ms", 10.0)
+ telemetry.set("memory.extract.stage.tool_skill_stats.duration_ms", 4.5)
+ telemetry.set("memory.extract.stage.profile_create.duration_ms", 7.0)
+ telemetry.set("memory.extract.stage.tool_skill_merge.duration_ms", 15.0)
+ telemetry.set("memory.extract.stage.dedup.duration_ms", 55.0)
+ telemetry.set("memory.extract.stage.create_memory.duration_ms", 12.0)
+ telemetry.set("memory.extract.stage.merge_existing.duration_ms", 9.0)
+ telemetry.set("memory.extract.stage.delete_existing.duration_ms", 0.0)
+ telemetry.set("memory.extract.stage.create_relations.duration_ms", 3.0)
+ telemetry.set("memory.extract.stage.flush_semantic.duration_ms", 1.0)
+ return {
+ "session_id": _session_id,
+ "status": "committed",
+ "memories_extracted": 2,
+ "active_count_updated": 0,
+ "archived": True,
+ "stats": None,
+ }
+
+ monkeypatch.setattr(service.sessions, "commit_async", fake_commit_async)
+
+ resp = await client.post(
+ f"/api/v1/sessions/{session_id}/commit",
+ json={"telemetry": True},
+ )
+ assert resp.status_code == 200
+ body = resp.json()
+ extract = body["telemetry"]["summary"]["memory"]["extract"]
+ assert extract["duration_ms"] == 321.5
+ assert extract["candidates"] == {"total": 4, "standard": 3, "tool_skill": 1}
+ assert extract["actions"] == {"created": 1, "merged": 1, "deleted": 0, "skipped": 2}
+ assert extract["stages"]["llm_extract_ms"] == 200.0
+ assert extract["stages"]["flush_semantic_ms"] == 1.0
+
+
async def test_compress_session_with_summary_only_telemetry(client: httpx.AsyncClient):
create_resp = await client.post("/api/v1/sessions", json={})
session_id = create_resp.json()["result"]["session_id"]
diff --git a/tests/telemetry/test_execution.py b/tests/telemetry/test_execution.py
index 12606553f..7b4afc25e 100644
--- a/tests/telemetry/test_execution.py
+++ b/tests/telemetry/test_execution.py
@@ -6,6 +6,150 @@ from openviking_cli.exceptions import InvalidArgumentError
from openviking_cli.retrieve.types import FindResult
+def test_operation_telemetry_summary_includes_memory_extract_breakdown():
+ from openviking.telemetry.operation import OperationTelemetry
+
+ telemetry = OperationTelemetry(operation="session.commit", enabled=True)
+ telemetry.set("memory.extracted", 5)
+ telemetry.set("memory.extract.total.duration_ms", 842.3)
+ telemetry.set("memory.extract.candidates.total", 7)
+ telemetry.set("memory.extract.candidates.standard", 5)
+ telemetry.set("memory.extract.candidates.tool_skill", 2)
+ telemetry.set("memory.extract.created", 3)
+ telemetry.set("memory.extract.merged", 1)
+ telemetry.set("memory.extract.deleted", 0)
+ telemetry.set("memory.extract.skipped", 3)
+ telemetry.set("memory.extract.stage.prepare_inputs.duration_ms", 8.4)
+ telemetry.set("memory.extract.stage.llm_extract.duration_ms", 410.2)
+ telemetry.set("memory.extract.stage.normalize_candidates.duration_ms", 6.7)
+ telemetry.set("memory.extract.stage.tool_skill_stats.duration_ms", 1.9)
+ telemetry.set("memory.extract.stage.profile_create.duration_ms", 12.5)
+ telemetry.set("memory.extract.stage.tool_skill_merge.duration_ms", 43.0)
+ telemetry.set("memory.extract.stage.dedup.duration_ms", 215.6)
+ telemetry.set("memory.extract.stage.create_memory.duration_ms", 56.1)
+ telemetry.set("memory.extract.stage.merge_existing.duration_ms", 22.7)
+ telemetry.set("memory.extract.stage.delete_existing.duration_ms", 0.0)
+ telemetry.set("memory.extract.stage.create_relations.duration_ms", 18.2)
+ telemetry.set("memory.extract.stage.flush_semantic.duration_ms", 9.0)
+
+ summary = telemetry.finish().summary
+
+ assert summary["memory"]["extracted"] == 5
+ assert summary["memory"]["extract"] == {
+ "duration_ms": 842.3,
+ "candidates": {
+ "total": 7,
+ "standard": 5,
+ "tool_skill": 2,
+ },
+ "actions": {
+ "created": 3,
+ "merged": 1,
+ "skipped": 3,
+ },
+ "stages": {
+ "prepare_inputs_ms": 8.4,
+ "llm_extract_ms": 410.2,
+ "normalize_candidates_ms": 6.7,
+ "tool_skill_stats_ms": 1.9,
+ "profile_create_ms": 12.5,
+ "tool_skill_merge_ms": 43.0,
+ "dedup_ms": 215.6,
+ "create_memory_ms": 56.1,
+ "merge_existing_ms": 22.7,
+ "create_relations_ms": 18.2,
+ "flush_semantic_ms": 9.0,
+ },
+ }
+
+
+def test_operation_telemetry_measure_accumulates_duration(monkeypatch):
+ from openviking.telemetry.operation import OperationTelemetry
+
+ perf_values = iter([10.0, 10.1, 10.3, 10.5, 10.8, 11.0])
+ monkeypatch.setattr(
+ "openviking.telemetry.operation.time.perf_counter", lambda: next(perf_values)
+ )
+
+ telemetry = OperationTelemetry(operation="session.commit", enabled=True)
+ with telemetry.measure("memory.extract.stage.dedup"):
+ pass
+ with telemetry.measure("memory.extract.stage.dedup"):
+ pass
+
+ summary = telemetry.finish().summary
+ assert summary["duration_ms"] == 1000.0
+ assert summary["memory"]["extract"]["stages"]["dedup_ms"] == 500.0
+
+
+def test_operation_telemetry_summary_includes_resource_breakdown():
+ from openviking.telemetry.operation import OperationTelemetry
+
+ telemetry = OperationTelemetry(operation="resources.add_resource", enabled=True)
+ telemetry.set("resource.request.duration_ms", 152.3)
+ telemetry.set("resource.process.duration_ms", 101.7)
+ telemetry.set("resource.parse.duration_ms", 38.1)
+ telemetry.set("resource.parse.warnings_count", 1)
+ telemetry.set("resource.finalize.duration_ms", 22.4)
+ telemetry.set("resource.summarize.duration_ms", 31.8)
+ telemetry.set("resource.wait.duration_ms", 46.9)
+ telemetry.set("resource.watch.duration_ms", 0.8)
+ telemetry.set("resource.flags.wait", True)
+ telemetry.set("resource.flags.build_index", True)
+ telemetry.set("resource.flags.summarize", False)
+ telemetry.set("resource.flags.watch_enabled", False)
+
+ summary = telemetry.finish().summary
+
+ assert summary["resource"] == {
+ "request": {"duration_ms": 152.3},
+ "process": {
+ "duration_ms": 101.7,
+ "parse": {"duration_ms": 38.1, "warnings_count": 1},
+ "finalize": {"duration_ms": 22.4},
+ "summarize": {"duration_ms": 31.8},
+ },
+ "wait": {"duration_ms": 46.9},
+ "watch": {"duration_ms": 0.8},
+ "flags": {
+ "wait": True,
+ "build_index": True,
+ "summarize": False,
+ "watch_enabled": False,
+ },
+ }
+
+
+def test_operation_telemetry_summary_omits_zero_valued_fields():
+ from openviking.telemetry.operation import OperationTelemetry
+
+ telemetry = OperationTelemetry(operation="resources.add_resource", enabled=True)
+ telemetry.set("queue.semantic.processed", 0)
+ telemetry.set("queue.semantic.error_count", 0)
+ telemetry.set("queue.embedding.processed", 4)
+ telemetry.set("queue.embedding.error_count", 0)
+ telemetry.set("semantic_nodes.total", 9)
+ telemetry.set("semantic_nodes.done", 8)
+ telemetry.set("semantic_nodes.pending", 1)
+ telemetry.set("semantic_nodes.running", 0)
+ telemetry.set("resource.process.duration_ms", 12.3)
+ telemetry.set("resource.parse.duration_ms", 0.0)
+ telemetry.set("resource.parse.warnings_count", 0)
+ telemetry.set("resource.flags.wait", False)
+ telemetry.set("resource.flags.build_index", True)
+
+ summary = telemetry.finish().summary
+
+ assert "tokens" not in summary
+ assert "semantic" not in summary["queue"]
+ assert summary["queue"]["embedding"] == {"processed": 4}
+ assert "running" not in summary["semantic_nodes"]
+ assert summary["resource"] == {
+ "process": {"duration_ms": 12.3},
+ "flags": {"wait": False, "build_index": True, "summarize": False, "watch_enabled": False},
+ }
+
+
@pytest.mark.asyncio
async def test_run_with_telemetry_returns_usage_and_payload():
from openviking.telemetry.execution import run_with_telemetry
diff --git a/tests/telemetry/test_resource_summary.py b/tests/telemetry/test_resource_summary.py
index 1d28e8b82..2fdd1cf01 100644
--- a/tests/telemetry/test_resource_summary.py
+++ b/tests/telemetry/test_resource_summary.py
@@ -55,4 +55,4 @@ def test_record_resource_wait_metrics_collects_queue_and_dag_stats(monkeypatch):
assert summary["semantic_nodes"]["total"] == 9
assert summary["semantic_nodes"]["done"] == 8
assert summary["semantic_nodes"]["pending"] == 1
- assert summary["semantic_nodes"]["running"] == 0
+ assert "running" not in summary["semantic_nodes"]
diff --git a/tests/test_telemetry_runtime.py b/tests/test_telemetry_runtime.py
index d3700706f..4a8fa8448 100644
--- a/tests/test_telemetry_runtime.py
+++ b/tests/test_telemetry_runtime.py
@@ -106,7 +106,6 @@ def test_telemetry_summary_uses_simplified_internal_metric_keys():
"total": 4,
"done": 3,
"pending": 1,
- "running": 0,
}
assert result["memory"] == {"extracted": 6}
@@ -171,7 +170,7 @@ async def test_semantic_processor_binds_registered_operation_telemetry(monkeypat
summary = result.summary
assert summary["tokens"]["total"] == 18
assert summary["tokens"]["llm"]["total"] == 18
- assert summary["tokens"]["embedding"]["total"] == 0
+ assert "embedding" not in summary["tokens"]
@pytest.mark.asyncio
@@ -281,13 +280,12 @@ async def test_resource_service_add_resource_reports_queue_summary(monkeypatch):
summary = telemetry_result.summary
assert summary["queue"] == {
"semantic": {"processed": 2, "error_count": 1},
- "embedding": {"processed": 5, "error_count": 0},
+ "embedding": {"processed": 5},
}
assert summary["semantic_nodes"] == {
"total": 3,
"done": 2,
"pending": 1,
- "running": 0,
}
assert "memory" not in summary
assert "errors" not in summary