Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(resource): add preserve_structure option for directory scanning (#509)

r266-tech8d9773675a9d
diff --git a/bot/tests/test_chat_functionality.py b/bot/tests/test_chat_functionality.py
index 276b463e1..27b353555 100644
--- a/bot/tests/test_chat_functionality.py
+++ b/bot/tests/test_chat_functionality.py
@@ -6,7 +6,6 @@ import tempfile
from pathlib import Path
import pytest
-
from vikingbot.bus.events import OutboundMessage
from vikingbot.bus.queue import MessageBus
from vikingbot.channels.chat import ChatChannel, ChatChannelConfig
diff --git a/bot/vikingbot/agent/context.py b/bot/vikingbot/agent/context.py
index 65453e92a..0af9b0aa6 100644
--- a/bot/vikingbot/agent/context.py
+++ b/bot/vikingbot/agent/context.py
@@ -3,11 +3,12 @@
import base64
import mimetypes
import platform
+import time as _time
+from datetime import datetime
from pathlib import Path
from typing import Any
+
from loguru import logger
-import time as _time
-from datetime import datetime
from vikingbot.agent.memory import MemoryStore
from vikingbot.agent.skills import SkillsLoader
diff --git a/bot/vikingbot/agent/loop.py b/bot/vikingbot/agent/loop.py
index e25609ed0..b72dd98f0 100644
--- a/bot/vikingbot/agent/loop.py
+++ b/bot/vikingbot/agent/loop.py
@@ -1,9 +1,12 @@
"""Agent loop: the core processing engine."""
+from __future__ import annotations
+
import asyncio
import json
import time
from pathlib import Path
+from typing import TYPE_CHECKING
from loguru import logger
@@ -24,6 +27,10 @@ from vikingbot.session.manager import SessionManager
from vikingbot.utils.helpers import cal_str_tokens
from vikingbot.utils.tracing import trace
+if TYPE_CHECKING:
+ from vikingbot.config.schema import ExecToolConfig
+ from vikingbot.cron.service import CronService
+
class AgentLoop:
"""
@@ -87,7 +94,7 @@ class AgentLoop:
... max_iterations=30,
... )
"""
- from vikingbot.config.schema import ExecToolConfig
+ from vikingbot.config.schema import ExecToolConfig # noqa: F811
self.bus = bus
self.provider = provider
@@ -305,7 +312,7 @@ class AgentLoop:
results = await asyncio.gather(*tool_tasks)
# Stage 3: Process results sequentially in original order
- for idx, tool_call, result, tool_execute_duration in results:
+ for _idx, tool_call, result, tool_execute_duration in results:
args_str = json.dumps(tool_call.arguments, ensure_ascii=False)
logger.info(f"[TOOL_CALL]: {tool_call.name}({args_str[:200]})")
logger.info(f"[RESULT]: {str(result)[:600]}")
diff --git a/bot/vikingbot/agent/memory.py b/bot/vikingbot/agent/memory.py
index f4909af7a..e7817839d 100644
--- a/bot/vikingbot/agent/memory.py
+++ b/bot/vikingbot/agent/memory.py
@@ -1,12 +1,11 @@
"""Memory system for persistent agent memory."""
-from openviking.async_client import logger
-from vikingbot.config.loader import load_config
from pathlib import Path
from typing import Any
from loguru import logger
import time
+from vikingbot.config.loader import load_config
from vikingbot.openviking_mount.ov_server import VikingClient
from vikingbot.utils.helpers import ensure_dir
diff --git a/bot/vikingbot/channels/dingtalk.py b/bot/vikingbot/channels/dingtalk.py
index c78ff64b1..dfb290330 100644
--- a/bot/vikingbot/channels/dingtalk.py
+++ b/bot/vikingbot/channels/dingtalk.py
@@ -5,8 +5,8 @@ import json
import time
from typing import Any
-from loguru import logger
import httpx
+from loguru import logger
from vikingbot.bus.events import OutboundMessage
from vikingbot.bus.queue import MessageBus
@@ -15,11 +15,11 @@ from vikingbot.config.schema import DingTalkChannelConfig
try:
from dingtalk_stream import (
- DingTalkStreamClient,
- Credential,
+ AckMessage,
CallbackHandler,
CallbackMessage,
- AckMessage,
+ Credential,
+ DingTalkStreamClient,
)
from dingtalk_stream.chatbot import ChatbotMessage
diff --git a/bot/vikingbot/channels/feishu.py b/bot/vikingbot/channels/feishu.py
index 00c3a8969..5c93c1b1e 100644
--- a/bot/vikingbot/channels/feishu.py
+++ b/bot/vikingbot/channels/feishu.py
@@ -35,17 +35,17 @@ from vikingbot.config.schema import FeishuChannelConfig
try:
import lark_oapi as lark
from lark_oapi.api.im.v1 import (
- CreateMessageRequest,
- CreateMessageRequestBody,
CreateMessageReactionRequest,
CreateMessageReactionRequestBody,
+ CreateMessageRequest,
+ CreateMessageRequestBody,
Emoji,
- P2ImMessageReceiveV1,
+ GetChatRequest,
GetImageRequest,
GetMessageResourceRequest,
+ P2ImMessageReceiveV1,
ReplyMessageRequest,
ReplyMessageRequestBody,
- GetChatRequest,
)
FEISHU_AVAILABLE = True
@@ -350,7 +350,10 @@ class FeishuChannel(BaseChannel):
lines = [l.strip() for l in table_text.strip().split("\n") if l.strip()]
if len(lines) < 3:
return None
- split = lambda l: [c.strip() for c in l.strip("|").split("|")]
+
+ def split(l: str) -> list[str]:
+ return [c.strip() for c in l.strip("|").split("|")]
+
headers = split(lines[0])
rows = [split(l) for l in lines[2:]]
columns = [
@@ -408,7 +411,6 @@ class FeishuChannel(BaseChannel):
before = protected[last_end : m.start()].strip()
if before:
elements.append({"tag": "markdown", "content": before})
- level = len(m.group(1))
text = m.group(2).strip()
elements.append(
{
@@ -715,7 +717,6 @@ class FeishuChannel(BaseChannel):
image_bytes = await self._download_feishu_image(image_key, message_id)
if image_bytes:
# Save to workspace/media directory
- from pathlib import Path
media_dir = get_data_path() / "received"
@@ -774,8 +775,8 @@ class FeishuChannel(BaseChannel):
},
)
- except Exception as e:
- logger.exception(f"Error processing Feishu message")
+ except Exception:
+ logger.exception("Error processing Feishu message")
async def _extract_and_upload_images(self, content: str) -> tuple[str, list[dict]]:
"""Extract images from markdown content, upload to Feishu, and return cleaned content."""
diff --git a/bot/vikingbot/channels/manager.py b/bot/vikingbot/channels/manager.py
index 8f6debe76..93a3aaa42 100644
--- a/bot/vikingbot/channels/manager.py
+++ b/bot/vikingbot/channels/manager.py
@@ -7,7 +7,6 @@ from typing import Any
from loguru import logger
-from vikingbot.bus.events import OutboundMessage
from vikingbot.bus.queue import MessageBus
from vikingbot.channels.base import BaseChannel
from vikingbot.config.schema import BaseChannelConfig, ChannelType, Config
diff --git a/bot/vikingbot/channels/openapi.py b/bot/vikingbot/channels/openapi.py
index d004522ed..45d0c848a 100644
--- a/bot/vikingbot/channels/openapi.py
+++ b/bot/vikingbot/channels/openapi.py
@@ -5,25 +5,21 @@ import secrets
import uuid
from datetime import datetime
from pathlib import Path
-from typing import Any, AsyncGenerator, Callable, Dict, List, Optional
+from typing import Any, Dict, List, Optional
-from fastapi import APIRouter, Depends, Header, HTTPException, Request
+from fastapi import APIRouter, Depends, FastAPI, Header, HTTPException
from fastapi.responses import StreamingResponse
from loguru import logger
-from pydantic import BaseModel
from vikingbot.bus.events import InboundMessage, OutboundEventType, OutboundMessage
from vikingbot.bus.queue import MessageBus
from vikingbot.channels.base import BaseChannel
from vikingbot.channels.openapi_models import (
- ChatMessage,
ChatRequest,
ChatResponse,
ChatStreamEvent,
- ErrorResponse,
EventType,
HealthResponse,
- MessageRole,
SessionCreateRequest,
SessionCreateResponse,
SessionDetailResponse,
diff --git a/bot/vikingbot/channels/qq.py b/bot/vikingbot/channels/qq.py
index c9decc01c..681da028d 100644
--- a/bot/vikingbot/channels/qq.py
+++ b/bot/vikingbot/channels/qq.py
@@ -2,7 +2,6 @@
import asyncio
from collections import deque
-from typing import TYPE_CHECKING
from loguru import logger
diff --git a/bot/vikingbot/cli/commands.py b/bot/vikingbot/cli/commands.py
index 41db22108..d380069da 100644
--- a/bot/vikingbot/cli/commands.py
+++ b/bot/vikingbot/cli/commands.py
@@ -4,9 +4,7 @@ import asyncio
import json
import os
import select
-import signal
import sys
-import hashlib
import time
from pathlib import Path
@@ -31,16 +29,15 @@ from vikingbot.cron.service import CronService
from vikingbot.cron.types import CronJob
from vikingbot.heartbeat.service import HeartbeatService
from vikingbot.integrations.langfuse import LangfuseClient
-from vikingbot.config.loader import load_config
# Create sandbox manager
from vikingbot.sandbox.manager import SandboxManager
from vikingbot.session.manager import SessionManager
from vikingbot.utils.helpers import (
+ get_bridge_path,
+ get_history_path,
get_source_workspace_path,
set_bot_data_path,
- get_history_path,
- get_bridge_path,
)
app = typer.Typer(
@@ -471,6 +468,7 @@ async def start_console(console_port):
"""Start the console web UI in a separate thread within the same process."""
try:
import threading
+
from vikingbot.console.console_gradio_simple import run_console_server
def run_in_thread():
@@ -794,8 +792,6 @@ def cron_list(
table.add_column("Status")
table.add_column("Next Run")
- import time
-
for job in jobs:
# Format schedule
if job.schedule.kind == "every":
diff --git a/bot/vikingbot/hooks/builtins/openviking_hooks.py b/bot/vikingbot/hooks/builtins/openviking_hooks.py
index 8eed926b3..50c1ecb6c 100644
--- a/bot/vikingbot/hooks/builtins/openviking_hooks.py
+++ b/bot/vikingbot/hooks/builtins/openviking_hooks.py
@@ -11,7 +11,6 @@ from ..base import Hook, HookContext
try:
import openviking as ov
-
from vikingbot.openviking_mount.ov_server import VikingClient
HAS_OPENVIKING = True
diff --git a/bot/vikingbot/integrations/langfuse.py b/bot/vikingbot/integrations/langfuse.py
index 9c53b3c1a..506cf2be0 100644
--- a/bot/vikingbot/integrations/langfuse.py
+++ b/bot/vikingbot/integrations/langfuse.py
@@ -94,12 +94,12 @@ class LangfuseClient:
user_id: Optional user ID to associate with all nested observations
"""
if not self.enabled:
- logger.warning(f"[LANGFUSE] propagate_attributes skipped: Langfuse client not enabled")
+ logger.warning("[LANGFUSE] propagate_attributes skipped: Langfuse client not enabled")
yield
return
if not self._client:
logger.warning(
- f"[LANGFUSE] propagate_attributes skipped: Langfuse client not initialized"
+ "[LANGFUSE] propagate_attributes skipped: Langfuse client not initialized"
)
yield
return
@@ -123,7 +123,7 @@ class LangfuseClient:
yield
else:
logger.warning(
- f"[LANGFUSE] propagate_attributes not available (SDK version may not support it)"
+ "[LANGFUSE] propagate_attributes not available (SDK version may not support it)"
)
yield
except Exception as e:
diff --git a/bot/vikingbot/openviking_mount/viking_fuse.py b/bot/vikingbot/openviking_mount/viking_fuse.py
index 13ec7d3c3..8714af23a 100644
--- a/bot/vikingbot/openviking_mount/viking_fuse.py
+++ b/bot/vikingbot/openviking_mount/viking_fuse.py
@@ -16,14 +16,14 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
from loguru import logger
-from .mount import OpenVikingMount, MountConfig
+from .mount import MountConfig, OpenVikingMount
# 尝试导入fusepy
try:
from fuse import FUSE, FuseOSError, Operations
FUSE_AVAILABLE = True
-except (ImportError, OSError) as e:
+except (ImportError, OSError):
FUSE_AVAILABLE = False
# 创建占位符
Operations = object
@@ -33,9 +33,9 @@ except (ImportError, OSError) as e:
# 只有当 FUSE 可用时才定义完整的实现
if FUSE_AVAILABLE:
+ import errno
import os
import stat
- import errno
from datetime import datetime
class OpenVikingFUSE(Operations):
@@ -405,7 +405,7 @@ if FUSE_AVAILABLE:
logger.info(f"Mounting OpenViking FUSE at: {config.mount_point}")
logger.info(f" Scope: {config.scope.value}")
logger.info(f" Read-only: {config.read_only}")
- logger.info(f" Press Ctrl+C to unmount")
+ logger.info(" Press Ctrl+C to unmount")
try:
FUSE(
diff --git a/bot/vikingbot/sandbox/backends/aiosandbox.py b/bot/vikingbot/sandbox/backends/aiosandbox.py
index 0265183fe..85d27e01d 100644
--- a/bot/vikingbot/sandbox/backends/aiosandbox.py
+++ b/bot/vikingbot/sandbox/backends/aiosandbox.py
@@ -1,15 +1,13 @@
"""AIO Sandbox backend implementation using agent-sandbox SDK."""
from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import Any
from loguru import logger
-from vikingbot.sandbox.base import SandboxBackend, SandboxNotStartedError
-from vikingbot.sandbox.backends import register_backend
-
-
from vikingbot.config.schema import SandboxConfig, SessionKey
+from vikingbot.sandbox.backends import register_backend
+from vikingbot.sandbox.base import SandboxBackend, SandboxNotStartedError
@register_backend("aiosandbox")
diff --git a/bot/vikingbot/sandbox/backends/opensandbox.py b/bot/vikingbot/sandbox/backends/opensandbox.py
index 8cab3c32f..4ef1d76af 100644
--- a/bot/vikingbot/sandbox/backends/opensandbox.py
+++ b/bot/vikingbot/sandbox/backends/opensandbox.py
@@ -1,22 +1,20 @@
"""OpenSandbox backend implementation using official SDK."""
+import asyncio
+import atexit
import os
import subprocess
-import asyncio
import time
-import atexit
from datetime import timedelta
from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import Any
import httpx
from loguru import logger
-from vikingbot.sandbox.base import SandboxBackend, SandboxNotStartedError
-from vikingbot.sandbox.backends import register_backend
-
-
from vikingbot.config.schema import SandboxConfig, SessionKey
+from vikingbot.sandbox.backends import register_backend
+from vikingbot.sandbox.base import SandboxBackend, SandboxNotStartedError
# Global to track the opensandbox-server process
_OSB_SERVER_PROCESS: "subprocess.Popen | None" = None
@@ -152,8 +150,8 @@ class OpenSandboxBackend(SandboxBackend):
)
try:
- from opensandbox.sandbox import Sandbox
from opensandbox.config import ConnectionConfig
+ from opensandbox.sandbox import Sandbox
self._connection_config = ConnectionConfig(
domain=self._server_url,
@@ -167,7 +165,7 @@ class OpenSandboxBackend(SandboxBackend):
volumes = None
if not self._is_vke:
# Local environment: mount host volume
- from opensandbox.models.sandboxes import Volume, Host
+ from opensandbox.models.sandboxes import Host, Volume
volumes = [
Volume(
diff --git a/bot/vikingbot/session/manager.py b/bot/vikingbot/session/manager.py
index 319039f95..32087f682 100644
--- a/bot/vikingbot/session/manager.py
+++ b/bot/vikingbot/session/manager.py
@@ -2,19 +2,16 @@
import asyncio
import json
-from pathlib import Path
from dataclasses import dataclass, field
-
from datetime import datetime
-from typing import TYPE_CHECKING, Any
+from pathlib import Path
+from typing import Any
from loguru import logger
from vikingbot.config.schema import SessionKey
-from vikingbot.utils.helpers import ensure_dir
-
-
from vikingbot.sandbox.manager import SandboxManager
+from vikingbot.utils.helpers import ensure_dir
@dataclass
diff --git a/openviking/parse/parsers/code/ast/languages/cpp.py b/openviking/parse/parsers/code/ast/languages/cpp.py
index b48c4993a..56a5ac900 100644
--- a/openviking/parse/parsers/code/ast/languages/cpp.py
+++ b/openviking/parse/parsers/code/ast/languages/cpp.py
@@ -134,7 +134,9 @@ def _extract_class(node, content_bytes: bytes, docstring: str = "") -> ClassSkel
return ClassSkeleton(name=name, bases=bases, docstring=docstring, methods=methods)
-def _extract_typedef_struct(node, content_bytes: bytes, docstring: str = "") -> Optional[ClassSkeleton]:
+def _extract_typedef_struct(
+ node, content_bytes: bytes, docstring: str = ""
+) -> Optional[ClassSkeleton]:
"""Handle typedef struct { ... } Name; and typedef struct Tag { ... } Name;
tree-sitter-cpp emits this as a 'type_definition' node with children:
@@ -159,7 +161,9 @@ def _extract_typedef_struct(node, content_bytes: bytes, docstring: str = "") ->
return skeleton if skeleton.name else None
-def _extract_function_proto(node, content_bytes: bytes, docstring: str = "") -> Optional[FunctionSig]:
+def _extract_function_proto(
+ node, content_bytes: bytes, docstring: str = ""
+) -> Optional[FunctionSig]:
"""Extract a function prototype from a top-level declaration node."""
fn = _extract_function(node, content_bytes, docstring=docstring)
return fn if fn.name else None
@@ -207,7 +211,7 @@ class CppExtractor(LanguageExtractor):
siblings = list(root.children)
top_level = []
- for idx, child in enumerate(siblings):
+ for _idx, child in enumerate(siblings):
if child.type == "preproc_include":
for sub in child.children:
if sub.type in ("string_literal", "system_lib_string"):
diff --git a/openviking/parse/parsers/directory.py b/openviking/parse/parsers/directory.py
index 8552b4434..da27b659d 100644
--- a/openviking/parse/parsers/directory.py
+++ b/openviking/parse/parsers/directory.py
@@ -106,6 +106,17 @@ class DirectoryParser(BaseParser):
exclude=kwargs.get("exclude"),
)
directly_upload_media = kwargs.get("directly_upload_media", True)
+ preserve_structure = kwargs.get("preserve_structure")
+ if preserve_structure is None:
+ # Fall back to config default
+ try:
+ from openviking_cli.utils.config.open_viking_config import (
+ get_openviking_config,
+ )
+
+ preserve_structure = get_openviking_config().directory.preserve_structure
+ except Exception:
+ preserve_structure = True
processable_files = scan_result.all_processable_files()
warnings.extend(scan_result.warnings)
@@ -157,6 +168,7 @@ class DirectoryParser(BaseParser):
target_uri,
viking_fs,
warnings,
+ preserve_structure=preserve_structure,
)
parser_name = "direct_upload"
else:
@@ -167,6 +179,7 @@ class DirectoryParser(BaseParser):
target_uri,
viking_fs,
warnings,
+ preserve_structure=preserve_structure,
)
if ok:
@@ -319,6 +332,7 @@ class DirectoryParser(BaseParser):
target_uri: str,
viking_fs: Any,
warnings: List[str],
+ preserve_structure: bool = True,
) -> bool:
"""Process one file into the VikingFS directory temp.
@@ -326,6 +340,11 @@ class DirectoryParser(BaseParser):
*target_uri* at the correct relative location.
- Files WITHOUT a parser → read and write directly to VikingFS.
+ Args:
+ preserve_structure: When True, files keep their relative directory
+ hierarchy. When False, all files are placed directly under
+ *target_uri* (flat).
+
Returns:
*True* on success, *False* on failure.
"""
@@ -336,8 +355,11 @@ class DirectoryParser(BaseParser):
try:
sub_result = await parser.parse(str(src_file))
if sub_result.temp_dir_path:
- parent = str(PurePosixPath(rel_path).parent)
- dest = f"{target_uri}/{parent}" if parent != "." else target_uri
+ if preserve_structure:
+ parent = str(PurePosixPath(rel_path).parent)
+ dest = f"{target_uri}/{parent}" if parent != "." else target_uri
+ else:
+ dest = target_uri
await DirectoryParser._merge_temp(
viking_fs,
sub_result.temp_dir_path,
@@ -350,7 +372,10 @@ class DirectoryParser(BaseParser):
else:
try:
content = src_file.read_bytes()
- dst_uri = f"{target_uri}/{rel_path}"
+ if preserve_structure:
+ dst_uri = f"{target_uri}/{rel_path}"
+ else:
+ dst_uri = f"{target_uri}/{PurePosixPath(rel_path).name}"
await viking_fs.write_file(dst_uri, content)
return True
except Exception as exc:
@@ -363,11 +388,17 @@ class DirectoryParser(BaseParser):
target_uri: str,
viking_fs: Any,
warnings: List[str],
+ preserve_structure: bool = True,
) -> bool:
"""Directly upload a file without using its parser.
Used for media files when directly_upload_media=True.
+ Args:
+ preserve_structure: When True, files keep their relative directory
+ hierarchy. When False, all files are placed directly under
+ *target_uri* (flat).
+
Returns:
*True* on success, *False* on failure.
"""
@@ -376,7 +407,10 @@ class DirectoryParser(BaseParser):
try:
content = src_file.read_bytes()
- dst_uri = f"{target_uri}/{rel_path}"
+ if preserve_structure:
+ dst_uri = f"{target_uri}/{rel_path}"
+ else:
+ dst_uri = f"{target_uri}/{PurePosixPath(rel_path).name}"
await viking_fs.write_file(dst_uri, content)
return True
except Exception as exc:
diff --git a/openviking/server/bootstrap.py b/openviking/server/bootstrap.py
index 9c45758ac..2665a0308 100644
--- a/openviking/server/bootstrap.py
+++ b/openviking/server/bootstrap.py
@@ -8,12 +8,11 @@ import shutil
import subprocess
import sys
import time
-
-import uvicorn
-
from dataclasses import dataclass
from typing import Optional
+import uvicorn
+
from openviking.server.app import create_app
from openviking.server.config import load_server_config
from openviking_cli.utils.logger import configure_uvicorn_logging
diff --git a/openviking/server/routers/bot.py b/openviking/server/routers/bot.py
index 8ec7b850e..d11cbf528 100644
--- a/openviking/server/routers/bot.py
+++ b/openviking/server/routers/bot.py
@@ -5,12 +5,11 @@ Vikingbot OpenAPIChannel when the --with-bot option is enabled.
"""
import json
-import os
from typing import AsyncGenerator, Optional
import httpx
-from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
-from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi import APIRouter, HTTPException, Request, status
+from fastapi.responses import StreamingResponse
from openviking_cli.utils.logger import get_logger
diff --git a/openviking/server/routers/resources.py b/openviking/server/routers/resources.py
index 6ba16028d..e1c63bc33 100644
--- a/openviking/server/routers/resources.py
+++ b/openviking/server/routers/resources.py
@@ -36,6 +36,7 @@ class AddResourceRequest(BaseModel):
include: Optional[str] = None
exclude: Optional[str] = None
directly_upload_media: bool = True
+ preserve_structure: Optional[bool] = None
@model_validator(mode="after")
def check_path_or_temp_path(self):
@@ -109,6 +110,16 @@ async def add_resource(
if path is None:
raise InvalidArgumentError("Either 'path' or 'temp_path' must be provided.")
+ kwargs = {
+ "strict": request.strict,
+ "ignore_dirs": request.ignore_dirs,
+ "include": request.include,
+ "exclude": request.exclude,
+ "directly_upload_media": request.directly_upload_media,
+ }
+ if request.preserve_structure is not None:
+ kwargs["preserve_structure"] = request.preserve_structure
+
result = await service.resources.add_resource(
path=path,
ctx=_ctx,
@@ -118,11 +129,7 @@ async def add_resource(
instruction=request.instruction,
wait=request.wait,
timeout=request.timeout,
- strict=request.strict,
- ignore_dirs=request.ignore_dirs,
- include=request.include,
- exclude=request.exclude,
- directly_upload_media=request.directly_upload_media,
+ **kwargs,
)
return Response(status="ok", result=result)
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index 1e918eb43..1dffc1c8a 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -6,10 +6,9 @@ Embedding utilities for OpenViking.
Common logic for creating Context objects and enqueuing them to EmbeddingQueue.
"""
-import asyncio
import os
from datetime import datetime
-from typing import Any, Dict, List, Optional, Union
+from typing import Dict, Optional
from openviking.core.context import Context, ContextLevel, ResourceContentType, Vectorize
from openviking.server.identity import RequestContext
diff --git a/openviking_cli/client/http.py b/openviking_cli/client/http.py
index 05507a5ed..a7e6b354a 100644
--- a/openviking_cli/client/http.py
+++ b/openviking_cli/client/http.py
@@ -294,6 +294,7 @@ class AsyncHTTPClient(BaseClient):
include: Optional[str] = None,
exclude: Optional[str] = None,
directly_upload_media: bool = True,
+ preserve_structure: Optional[bool] = None,
) -> Dict[str, Any]:
"""Add resource to OpenViking."""
# Validate that only one of 'to' or 'parent' is set
@@ -313,6 +314,8 @@ class AsyncHTTPClient(BaseClient):
"exclude": exclude,
"directly_upload_media": directly_upload_media,
}
+ if preserve_structure is not None:
+ request_data["preserve_structure"] = preserve_structure
path_obj = Path(path)
if path_obj.exists() and not self._is_local_server():
diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py
index 723425634..9745f0171 100644
--- a/openviking_cli/utils/config/open_viking_config.py
+++ b/openviking_cli/utils/config/open_viking_config.py
@@ -21,6 +21,7 @@ from .log_config import LogConfig
from .parser_config import (
AudioConfig,
CodeConfig,
+ DirectoryConfig,
HTMLConfig,
ImageConfig,
MarkdownConfig,
@@ -89,6 +90,10 @@ class OpenVikingConfig(BaseModel):
default_factory=lambda: TextConfig(), description="Text parsing configuration"
)
+ directory: DirectoryConfig = Field(
+ default_factory=lambda: DirectoryConfig(), description="Directory parsing configuration"
+ )
+
auto_generate_l0: bool = Field(
default=True, description="Automatically generate L0 (abstract) if not provided"
)
@@ -136,7 +141,17 @@ class OpenVikingConfig(BaseModel):
parser_configs = {}
if "parsers" in config_copy:
parser_configs = config_copy.pop("parsers")
- parser_types = ["pdf", "code", "image", "audio", "video", "markdown", "html", "text"]
+ parser_types = [
+ "pdf",
+ "code",
+ "image",
+ "audio",
+ "video",
+ "markdown",
+ "html",
+ "text",
+ "directory",
+ ]
for parser_type in parser_types:
if parser_type in config_copy:
parser_configs[parser_type] = config_copy.pop(parser_type)
diff --git a/openviking_cli/utils/config/parser_config.py b/openviking_cli/utils/config/parser_config.py
index f59b2fa33..2ec28f735 100644
--- a/openviking_cli/utils/config/parser_config.py
+++ b/openviking_cli/utils/config/parser_config.py
@@ -465,6 +465,21 @@ class TextConfig(ParserConfig):
raise ValueError("max_paragraph_length must be positive")
+@dataclass
+class DirectoryConfig(ParserConfig):
+ """
+ Configuration for directory parsing.
+
+ Attributes:
+ preserve_structure: Whether to preserve nested directory structure when
+ adding directory resources. When True (default), files maintain their
+ relative path hierarchy. When False, all files are flattened to a
+ single level under the resource root.
+ """
+
+ preserve_structure: bool = True
+
+
# Configuration registry for dynamic loading
PARSER_CONFIG_REGISTRY = {
"pdf": PDFConfig,
@@ -475,6 +490,7 @@ PARSER_CONFIG_REGISTRY = {
"markdown": MarkdownConfig,
"html": HTMLConfig,
"text": TextConfig,
+ "directory": DirectoryConfig,
}
@@ -505,10 +521,8 @@ def get_parser_config(
... })
"""
if parser_type not in PARSER_CONFIG_REGISTRY:
- raise ValueError(
- f"Unsupported parser type: '{parser_type}'. "
- f"Supported types: {list(PARSER_CONFIG_REGISTRY.keys())}"
- )
+ supported = list(PARSER_CONFIG_REGISTRY.keys())
+ raise ValueError(f"Unsupported parser type: '{parser_type}'. Supported: {supported}")
config_class = PARSER_CONFIG_REGISTRY[parser_type]
diff --git a/tests/parse/test_add_directory.py b/tests/parse/test_add_directory.py
index 5c4e884b5..492f1c8d8 100644
--- a/tests/parse/test_add_directory.py
+++ b/tests/parse/test_add_directory.py
@@ -768,3 +768,76 @@ class TestDirectlyUploadMedia:
assert mock_image.parse.call_count == 2
mock_audio.parse.assert_called_once()
mock_video.parse.assert_called_once()
+
+
+# ---------------------------------------------------------------------------
+# Tests: preserve_structure parameter
+# ---------------------------------------------------------------------------
+
+
+class TestPreserveStructure:
+ """Tests for the preserve_structure parameter."""
+
+ @pytest.mark.asyncio
+ async def test_preserve_structure_true_keeps_hierarchy(
+ self, tmp_nested_code: Path, parser, fake_fs
+ ) -> None:
+ """preserve_structure=True (default) keeps relative directory paths."""
+ await parser.parse(str(tmp_nested_code), preserve_structure=True)
+
+ dir_name = tmp_nested_code.name
+ rel_paths = set()
+ for uri in fake_fs.files:
+ idx = uri.find(f"/{dir_name}/")
+ if idx >= 0:
+ rel = uri[idx + len(f"/{dir_name}/") :]
+ rel_paths.add(rel)
+
+ assert "top.py" in rel_paths
+ assert "a/x.py" in rel_paths
+ assert "a/b/c.py" in rel_paths
+ assert "a/b/d.py" in rel_paths
+
+ @pytest.mark.asyncio
+ async def test_preserve_structure_false_flattens(
+ self, tmp_nested_code: Path, parser, fake_fs
+ ) -> None:
+ """preserve_structure=False flattens all files to a single level."""
+ await parser.parse(str(tmp_nested_code), preserve_structure=False)
+
+ dir_name = tmp_nested_code.name
+ rel_paths = set()
+ for uri in fake_fs.files:
+ idx = uri.find(f"/{dir_name}/")
+ if idx >= 0:
+ rel = uri[idx + len(f"/{dir_name}/") :]
+ rel_paths.add(rel)
+
+ # All files should be at the top level (no '/' in paths)
+ for rel in rel_paths:
+ assert "/" not in rel, f"Expected flat path, got: {rel}"
+
+ # All filenames should be present
+ assert "top.py" in rel_paths
+ assert "x.py" in rel_paths
+ assert "c.py" in rel_paths
+ assert "d.py" in rel_paths
+
+ @pytest.mark.asyncio
+ async def test_preserve_structure_default_is_true(
+ self, tmp_nested_code: Path, parser, fake_fs
+ ) -> None:
+ """Default behavior (no explicit param) preserves structure."""
+ await parser.parse(str(tmp_nested_code))
+
+ dir_name = tmp_nested_code.name
+ rel_paths = set()
+ for uri in fake_fs.files:
+ idx = uri.find(f"/{dir_name}/")
+ if idx >= 0:
+ rel = uri[idx + len(f"/{dir_name}/") :]
+ rel_paths.add(rel)
+
+ # Structure should be preserved by default
+ assert "a/b/c.py" in rel_paths
+ assert "a/x.py" in rel_paths
diff --git a/tests/parse/test_ast_extractor.py b/tests/parse/test_ast_extractor.py
index 99c17d3a7..d02517156 100644
--- a/tests/parse/test_ast_extractor.py
+++ b/tests/parse/test_ast_extractor.py
@@ -2,10 +2,8 @@
# SPDX-License-Identifier: Apache-2.0
"""Tests for AST-based code skeleton extraction."""
-import pytest
from openviking.parse.parsers.code.ast.skeleton import ClassSkeleton, CodeSkeleton, FunctionSig
-
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------