Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat: concurrent embedding, GitHub ZIP download, read offset/limit, code parser optimization (#267)
yangxinxin-77557f5d504c4
diff --git a/openviking/async_client.py b/openviking/async_client.pyindex 1dc1f1a87..cfc3920fb 100644--- a/openviking/async_client.py+++ b/openviking/async_client.py@@ -255,10 +255,10 @@ class AsyncOpenViking:await self._ensure_initialized()return await self._client.overview(uri)- async def read(self, uri: str) -> str:+ async def read(self, uri: str, offset: int = 0, limit: int = -1) -> str:"""Read file content"""await self._ensure_initialized()- return await self._client.read(uri)+ return await self._client.read(uri, offset=offset, limit=limit)async def ls(self, uri: str, **kwargs) -> List[Any]:"""diff --git a/openviking/client/local.py b/openviking/client/local.pyindex 208d7312b..27ce39c77 100644--- a/openviking/client/local.py+++ b/openviking/client/local.py@@ -144,9 +144,9 @@ class LocalClient(BaseClient):# ============= Content Reading =============- async def read(self, uri: str) -> str:+ async def read(self, uri: str, offset: int = 0, limit: int = -1) -> str:"""Read file content."""- return await self._service.fs.read(uri)+ return await self._service.fs.read(uri, offset=offset, limit=limit)async def abstract(self, uri: str) -> str:"""Read L0 abstract."""diff --git a/openviking/parse/parsers/code/code.py b/openviking/parse/parsers/code/code.pyindex f22f0a421..3693384b6 100644--- a/openviking/parse/parsers/code/code.py+++ b/openviking/parse/parsers/code/code.py@@ -15,6 +15,8 @@ import shutilimport statimport tempfileimport time+import urllib.request+import zipfilefrom pathlib import Path, PurePosixPathfrom typing import Any, List, Optional, Tuple, Unionfrom urllib.parse import unquote, urlparse@@ -106,14 +108,22 @@ class CodeRepositoryParser(BaseParser):# 2. Fetch content (Clone or Extract)repo_name = "repository"+ local_dir = Path(temp_local_dir)if source_str.startswith(("http://", "https://", "git://", "ssh://")):repo_url, branch, commit = self._parse_repo_source(source_str, **kwargs)- repo_name = await self._git_clone(- repo_url,- temp_local_dir,- branch=branch,- commit=commit,- )+ if self._is_github_url(repo_url) and not commit:+ # Use GitHub ZIP API: single HTTPS download, no git history, much faster+ local_dir, repo_name = await self._github_zip_download(+ repo_url, branch, temp_local_dir+ )+ else:+ # Non-GitHub URL or specific commit: fall back to git clone+ repo_name = await self._git_clone(+ repo_url,+ temp_local_dir,+ branch=branch,+ commit=commit,+ )elif str(source).endswith(".zip"):repo_name = await self._extract_zip(source_str, temp_local_dir)else:@@ -128,9 +138,7 @@ class CodeRepositoryParser(BaseParser):logger.info(f"Uploading to VikingFS: {target_root_uri}")# 4. Upload to VikingFS (filtering on the fly)- file_count = await self._upload_directory(- Path(temp_local_dir), target_root_uri, viking_fs- )+ file_count = await self._upload_directory(local_dir, target_root_uri, viking_fs)logger.info(f"Uploaded {file_count} files to {target_root_uri}")@@ -185,9 +193,7 @@ class CodeRepositoryParser(BaseParser):"""Not supported for repositories."""raise NotImplementedError("CodeRepositoryParser does not support parse_content")- def _parse_repo_source(- self, source: str, **kwargs- ) -> Tuple[str, Optional[str], Optional[str]]:+ def _parse_repo_source(self, source: str, **kwargs) -> Tuple[str, Optional[str], Optional[str]]:branch = kwargs.get("branch") or kwargs.get("ref")commit = kwargs.get("commit")repo_url = source@@ -275,6 +281,91 @@ class CodeRepositoryParser(BaseParser):except RuntimeError:return False+ @staticmethod+ def _is_github_url(url: str) -> bool:+ """Return True for github.com URLs (supports ZIP archive API)."""+ return urlparse(url).netloc in ("github.com", "www.github.com")++ async def _github_zip_download(+ self,+ repo_url: str,+ branch: Optional[str],+ target_dir: str,+ ) -> Tuple[Path, str]:+ """Download a GitHub repo as a ZIP archive and extract it.++ Uses the GitHub archive API (single HTTPS GET, no git history).++ Returns:+ (content_dir, repo_name) — content_dir is the extracted repo root.+ """+ repo_name = self._get_repo_name(repo_url)++ # Build archive URL from owner/repo path components.+ parsed = urlparse(repo_url)+ path_parts = [p for p in parsed.path.split("/") if p]+ owner = path_parts[0]+ repo_raw = path_parts[1]+ # Strip .git suffix for the archive URL (git clone keeps it, ZIP API does not).+ repo_slug = repo_raw[:-4] if repo_raw.endswith(".git") else repo_raw++ if branch:+ zip_url = f"https://github.com/{owner}/{repo_slug}/archive/refs/heads/{branch}.zip"+ else:+ zip_url = f"https://github.com/{owner}/{repo_slug}/archive/HEAD.zip"++ logger.info(f"Downloading GitHub ZIP: {zip_url}")++ zip_path = os.path.join(target_dir, "_archive.zip")+ extract_dir = os.path.join(target_dir, "_extracted")+ os.makedirs(extract_dir, exist_ok=True)++ # Download (blocking HTTP; run in thread pool to avoid stalling event loop).+ def _download() -> None:+ req = urllib.request.Request(zip_url, headers={"User-Agent": "OpenViking"})+ with urllib.request.urlopen(req) as resp, open(zip_path, "wb") as f:+ shutil.copyfileobj(resp, f)++ try:+ await asyncio.to_thread(_download)+ except Exception as exc:+ raise RuntimeError(f"Failed to download GitHub ZIP {zip_url}: {exc}")++ # Safe extraction with Zip Slip validation (mirrors _extract_zip logic).+ target = Path(extract_dir).resolve()+ with zipfile.ZipFile(zip_path, "r") as zf:+ for info in zf.infolist():+ mode = info.external_attr >> 16+ if info.is_dir() or stat.S_ISDIR(mode):+ continue+ if stat.S_ISLNK(mode):+ logger.warning(f"Skipping symlink entry in GitHub ZIP: {info.filename}")+ continue+ raw = info.filename.replace("\\", "/")+ raw_parts = [p for p in raw.split("/") if p]+ if ".." in raw_parts:+ raise ValueError(f"Zip Slip detected in GitHub archive: {info.filename!r}")+ if PurePosixPath(raw).is_absolute():+ raise ValueError(f"Zip Slip detected in GitHub archive: {info.filename!r}")+ extracted = Path(zf.extract(info, extract_dir)).resolve()+ if not extracted.is_relative_to(target):+ extracted.unlink(missing_ok=True)+ raise ValueError(f"Zip Slip detected in GitHub archive: {info.filename!r}")++ # Remove downloaded archive to free disk space.+ try:+ os.unlink(zip_path)+ except OSError:+ pass++ # GitHub ZIPs have a single top-level directory: {repo}-{branch}/ or {repo}-{sha}/.+ # Return that directory as the content root so callers see bare repo files.+ top_level = [d for d in Path(extract_dir).iterdir() if d.is_dir()]+ content_dir = top_level[0] if len(top_level) == 1 else Path(extract_dir)++ logger.info(f"GitHub ZIP extracted to {content_dir} ({repo_name})")+ return content_dir, repo_name+async def _git_clone(self,url: str,@@ -282,26 +373,15 @@ class CodeRepositoryParser(BaseParser):branch: Optional[str] = None,commit: Optional[str] = None,) -> str:- """- Clone git repository.+ """Clone a git repository into target_dir; return the repo name.++ Uses --depth 1 for speed. If a specific commit is requested, it is+ fetched and checked out after the shallow clone.Returns:- Repository name (e.g. "OpenViking" from "https://.../OpenViking.git")+ Repository name derived from the URL (e.g. "OpenViking")."""- # Extract repo name from URLname = self._get_repo_name(url)-- # Clone into a subdirectory to keep structure clean- # But here we clone content directly into target_dir?- # Actually, git clone <url> <dir> clones INTO <dir>.- # But if we want the repo name directory to exist in VikingFS, we should clone into target_dir/name?- # No, parse logic says:- # temp_local_dir contains the files (e.g. .git, src, README)- # We upload temp_local_dir content to viking://temp/{uuid}/{repo_name}/-- # So we clone current content directly into temp_local_dir- # git clone --depth 1 url target_dir-logger.info(f"Cloning {url} to {target_dir}...")clone_args = [@@ -320,19 +400,30 @@ class CodeRepositoryParser(BaseParser):await self._run_git(["git", "-C", target_dir, "fetch", "origin", commit])except RuntimeError:try:- await self._run_git(["git", "-C", target_dir, "fetch", "--all", "--tags", "--prune"])+ await self._run_git(+ ["git", "-C", target_dir, "fetch", "--all", "--tags", "--prune"]+ )except RuntimeError:passok = await self._has_commit(target_dir, commit)if not ok:try:- await self._run_git(["git", "-C", target_dir, "fetch", "--unshallow", "origin"])+ await self._run_git(+ ["git", "-C", target_dir, "fetch", "--unshallow", "origin"]+ )except RuntimeError:passok = await self._has_commit(target_dir, commit)if not ok:await self._run_git(- ["git", "-C", target_dir, "fetch", "origin", "+refs/heads/*:refs/remotes/origin/*"]+ [+ "git",+ "-C",+ target_dir,+ "fetch",+ "origin",+ "+refs/heads/*:refs/remotes/origin/*",+ ])ok = await self._has_commit(target_dir, commit)if not ok:@@ -342,17 +433,7 @@ class CodeRepositoryParser(BaseParser):return nameasync def _extract_zip(self, zip_path: str, target_dir: str) -> str:- """Extract zip file."""- import zipfile-- # We assume it's a local path if passed here?- # Actually logic in parse() handles local path check before calling here?- # Or if it's a URL ending in zip, HTMLParser might have downloaded it?- # Wait, HTMLParser handles download. If we are here, source IS a path or URL.- # If it's a URL, we need to download it first?- # CodeRepositoryParser is designed to handle "source" which can be URL.- # So I need to download zip if it is a URL.-+ """Extract a local zip file into target_dir; return the archive stem as the repo name."""if zip_path.startswith(("http://", "https://")):# TODO: implement download logic or rely on caller?# For now, assume it's implemented if needed, but raise error as strictly we only support git URL for now as per plandiff --git a/openviking/parse/parsers/upload_utils.py b/openviking/parse/parsers/upload_utils.pyindex 39f73744f..50c80ed02 100644--- a/openviking/parse/parsers/upload_utils.py+++ b/openviking/parse/parsers/upload_utils.py@@ -2,6 +2,7 @@# SPDX-License-Identifier: Apache-2.0"""Shared upload utilities for directory and file uploading to VikingFS."""+import asyncioimport osimport refrom pathlib import Path@@ -65,11 +66,13 @@ def detect_and_convert_encoding(content: bytes, file_path: Union[str, Path] = ""# Check for potential binary content (null bytes in first 8KB)# Binary files often contain null bytes which can cause issuessample_size = min(8192, len(content))- if b'\x00' in content[:sample_size]:- null_count = content[:sample_size].count(b'\x00')+ if b"\x00" in content[:sample_size]:+ null_count = content[:sample_size].count(b"\x00")# If more than 5% null bytes in sample, likely binary - don't processif null_count / sample_size > 0.05:- logger.debug(f"Detected binary content in {file_path} (null bytes: {null_count}), skipping encoding detection")+ logger.debug(+ f"Detected binary content in {file_path} (null bytes: {null_count}), skipping encoding detection"+ )return contentdetected_encoding: Optional[str] = None@@ -77,7 +80,7 @@ def detect_and_convert_encoding(content: bytes, file_path: Union[str, Path] = ""try:decoded = content.decode(encoding)# Additional validation: check for control characters that suggest binary- control_chars = sum(1 for c in decoded[:1000] if ord(c) < 32 and c not in '\t\n\r')+ control_chars = sum(1 for c in decoded[:1000] if ord(c) < 32 and c not in "\t\n\r")if control_chars / min(1000, len(decoded)) > 0.05: # More than 5% control charscontinuedetected_encoding = encoding@@ -92,8 +95,8 @@ def detect_and_convert_encoding(content: bytes, file_path: Union[str, Path] = ""if detected_encoding not in UTF8_VARIANTS:decoded_content = content.decode(detected_encoding, errors="replace")# Remove null bytes from decoded content as they can cause issues downstream- if '\x00' in decoded_content:- decoded_content = decoded_content.replace('\x00', '')+ if "\x00" in decoded_content:+ decoded_content = decoded_content.replace("\x00", "")logger.debug(f"Removed null bytes from decoded content in {file_path}")content = decoded_content.encode("utf-8")logger.debug(f"Converted {file_path} from {detected_encoding} to UTF-8")@@ -195,6 +198,9 @@ async def upload_text_files(return uploaded_count, warnings+_UPLOAD_CONCURRENCY = 8++async def upload_directory(local_dir: Path,viking_uri_base: str,@@ -203,24 +209,26 @@ async def upload_directory(ignore_extensions: Optional[Set[str]] = None,max_file_size: int = 10 * 1024 * 1024,) -> Tuple[int, List[str]]:- """Upload an entire directory recursively and return uploaded count with warnings."""+ """Upload an entire directory recursively and return uploaded count with warnings.++ Optimized: collects all files in one pass, pre-creates directories upfront,+ then uploads all files concurrently (up to _UPLOAD_CONCURRENCY at a time).+ """effective_ignore_dirs = ignore_dirs if ignore_dirs is not None else IGNORE_DIRSeffective_ignore_extensions = (ignore_extensions if ignore_extensions is not None else IGNORE_EXTENSIONS)- uploaded_count = 0warnings: List[str] = []- await viking_fs.mkdir(viking_uri_base, exist_ok=True)+ # --- Phase 1: Collect files and unique parent directory URIs in one pass ---+ files_to_upload: List[Tuple[Path, str]] = [] # (local_path, target_uri)+ parent_uris: Set[str] = {viking_uri_base}for root, dirs, files in os.walk(local_dir):dirs[:] = [- dir_name- for dir_name in dirs- if not should_skip_directory(dir_name, ignore_dirs=effective_ignore_dirs)+ d for d in dirs if not should_skip_directory(d, ignore_dirs=effective_ignore_dirs)]-for file_name in files:file_path = Path(root) / file_nameshould_skip, _ = should_skip_file(@@ -230,18 +238,69 @@ async def upload_directory()if should_skip:continue-rel_path_str = str(file_path.relative_to(local_dir)).replace(os.sep, "/")try:safe_rel = _sanitize_rel_path(rel_path_str)- target_uri = f"{viking_uri_base}/{safe_rel}"- content = file_path.read_bytes()- content = detect_and_convert_encoding(content, file_path)- await viking_fs.write_file_bytes(target_uri, content)- uploaded_count += 1- except Exception as exc:- warning = f"Failed to upload {file_path}: {exc}"+ except ValueError as exc:+ warning = f"Skipping {file_path}: {exc}"warnings.append(warning)logger.warning(warning)+ continue+ target_uri = f"{viking_uri_base}/{safe_rel}"+ files_to_upload.append((file_path, target_uri))+ parent_uris.add(target_uri.rsplit("/", 1)[0])++ # --- Phase 2: Pre-create all directories ---+ # Memoized mkdir: each unique agfs path is created at most once.+ # This is equivalent to _ensure_parent_dirs but avoids redundant HTTP calls+ # by tracking already-processed paths across all directories.+ _created: Set[str] = set()++ def _mkdir_with_parents(agfs_path: str) -> None:+ parts = agfs_path.lstrip("/").split("/")+ for i in range(1, len(parts) + 1):+ p = "/" + "/".join(parts[:i])+ if p in _created:+ continue+ try:+ viking_fs.agfs.mkdir(p)+ _created.add(p)+ except Exception as e:+ if "already" in str(e).lower():+ _created.add(p)+ else:+ logger.warning(f"Failed to create directory {p}: {e}")++ def _create_all_dirs() -> None:+ for dir_uri in sorted(parent_uris):+ _mkdir_with_parents(viking_fs._uri_to_path(dir_uri))++ await asyncio.to_thread(_create_all_dirs)++ # --- Phase 3: Upload files concurrently ---+ sem = asyncio.Semaphore(_UPLOAD_CONCURRENCY)+ errors: List[Optional[str]] = [None] * len(files_to_upload)++ async def _upload_one(idx: int, file_path: Path, target_uri: str) -> None:+ async with sem:++ def _do() -> None:+ content = file_path.read_bytes()+ encoded = detect_and_convert_encoding(content, file_path)+ agfs_path = viking_fs._uri_to_path(target_uri)+ viking_fs.agfs.write(agfs_path, encoded)++ try:+ await asyncio.to_thread(_do)+ except Exception as exc:+ errors[idx] = f"Failed to upload {file_path}: {exc}"++ await asyncio.gather(*[_upload_one(i, fp, uri) for i, (fp, uri) in enumerate(files_to_upload)])++ for err in errors:+ if err:+ warnings.append(err)+ logger.warning(err)+ uploaded_count = sum(1 for e in errors if e is None)return uploaded_count, warningsdiff --git a/openviking/parse/tree_builder.py b/openviking/parse/tree_builder.pyindex 7d3f5a891..2d202aa01 100644--- a/openviking/parse/tree_builder.py+++ b/openviking/parse/tree_builder.py@@ -20,6 +20,7 @@ IMPORTANT (v5.0 Architecture):- Content splitting is handled by Parser, not TreeBuilder"""+import asyncioimport loggingfrom typing import TYPE_CHECKING, Optional@@ -139,7 +140,7 @@ class TreeBuilder:logger.info(f"[TreeBuilder] Finalizing from temp: {final_uri}")# 4. Move directory tree from temp to final location in AGFS- await self._move_directory_in_agfs(temp_doc_uri, final_uri)+ await self._move_temp_to_dest(viking_fs, temp_doc_uri, final_uri)logger.info(f"[TreeBuilder] Moved temp tree: {temp_doc_uri} -> {final_uri}")# 5. Cleanup temporary root directory@@ -191,39 +192,15 @@ class TreeBuilder:raise FileExistsError(f"Cannot resolve unique name for {uri} after {max_attempts} attempts")- async def _move_directory_in_agfs(self, src_uri: str, dst_uri: str) -> None:- """Recursively move AGFS directory tree (copy + delete)."""- viking_fs = get_viking_fs()+ async def _move_temp_to_dest(self, viking_fs, src_uri: str, dst_uri: str) -> None:+ """Move temp directory to final destination using a single native AGFS mv call.- # 1. Ensure parent directories exist+ Temp files have no vector records yet, so no vector index update is needed.+ """+ src_path = viking_fs._uri_to_path(src_uri)+ dst_path = viking_fs._uri_to_path(dst_uri)await self._ensure_parent_dirs(dst_uri)-- # 2. Create target directory- await viking_fs.mkdir(dst_uri)-- # 3. List source directory contents- entries = await viking_fs.ls(src_uri)-- for entry in entries:- name = entry.get("name", "")- if not name or name in [".", ".."]:- continue-- src_item = f"{src_uri}/{name}"- dst_item = f"{dst_uri}/{name}"-- if entry.get("isDir"):- # Recursively move subdirectory- await self._move_directory_in_agfs(src_item, dst_item)- else:- # Move file- await viking_fs.move_file(src_item, dst_item)-- # 4. Delete source directory (should be empty now)- try:- await viking_fs.rm(src_uri)- except Exception:- pass # Ignore error when deleting empty directory+ await asyncio.to_thread(viking_fs.agfs.mv, src_path, dst_path)async def _ensure_parent_dirs(self, uri: str) -> None:"""Recursively create parent directories."""diff --git a/openviking/server/routers/content.py b/openviking/server/routers/content.pyindex c1e9d8cef..38a3cde35 100644--- a/openviking/server/routers/content.py+++ b/openviking/server/routers/content.py@@ -15,11 +15,13 @@ router = APIRouter(prefix="/api/v1/content", tags=["content"])@router.get("/read")async def read(uri: str = Query(..., description="Viking URI"),+ offset: int = Query(0, description="Starting line number (0-indexed)"),+ limit: int = Query(-1, description="Number of lines to read, -1 means read to end"),_ctx: RequestContext = Depends(get_request_context),):"""Read file content (L2)."""service = get_service()- result = await service.fs.read(uri)+ result = await service.fs.read(uri, offset=offset, limit=limit)return Response(status="ok", result=result)diff --git a/openviking/service/core.py b/openviking/service/core.pyindex b040c24fb..59d81163a 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -89,7 +89,7 @@ class OpenVikingService:self._initialized = False# Initialize storage- self._init_storage(config.storage)+ self._init_storage(config.storage, config.embedding.max_concurrent)# Initialize embedderself._embedder = config.embedding.get_embedder()@@ -97,7 +97,7 @@ class OpenVikingService:f"Initialized embedder (dim {config.embedding.dimension}, sparse {self._embedder.is_sparse})")- def _init_storage(self, config: StorageConfig) -> None:+ def _init_storage(self, config: StorageConfig, max_concurrent_embedding: int = 1) -> None:"""Initialize storage resources."""if config.agfs.backend == "local":self._agfs_manager = AGFSManager(config=config.agfs)@@ -112,6 +112,7 @@ class OpenVikingService:self._queue_manager = init_queue_manager(agfs_url=self._agfs_url,timeout=config.agfs.timeout,+ max_concurrent_embedding=max_concurrent_embedding,)else:logger.warning("AGFS URL not configured, skipping queue manager initialization")@@ -195,7 +196,7 @@ class OpenVikingService:returnif self._vikingdb_manager is None:- self._init_storage(self._config.storage)+ self._init_storage(self._config.storage, self._config.embedding.max_concurrent)if self._embedder is None:self._embedder = self._config.embedding.get_embedder()@@ -204,10 +205,10 @@ class OpenVikingService:# Initialize VikingFS and VikingDB with recorder if enabledenable_recorder = os.environ.get("OPENVIKING_ENABLE_RECORDER", "").lower() == "true"-+# Create context collectionawait init_context_collection(self._vikingdb_manager)-+self._viking_fs = init_viking_fs(agfs_url=self._agfs_url or "http://localhost:8080",query_embedder=self._embedder,diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.pyindex c57afb1e8..4f5dcc455 100644--- a/openviking/service/fs_service.py+++ b/openviking/service/fs_service.py@@ -121,10 +121,10 @@ class FSService:viking_fs = self._ensure_initialized()return await viking_fs.stat(uri)- async def read(self, uri: str) -> str:+ async def read(self, uri: str, offset: int = 0, limit: int = -1) -> str:"""Read file content."""viking_fs = self._ensure_initialized()- return await viking_fs.read_file(uri)+ return await viking_fs.read_file(uri, offset=offset, limit=limit)async def abstract(self, uri: str) -> str:"""Read L0 abstract (.abstract.md)."""diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex 584cc42b1..c703b33bd 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -7,6 +7,7 @@ Provides centralized schema definitions and factory functions for creating collesimilar to how init_viking_fs encapsulates VikingFS initialization."""+import asyncioimport hashlibimport jsonfrom typing import Any, Dict, Optional@@ -149,7 +150,11 @@ class TextEmbeddingHandler(DequeueHandlerBase):# Generate embedding vector(s)if self._embedder:- result: EmbedResult = self._embedder.embed(embedding_msg.message)+ # embed() is a blocking HTTP call; offload to thread pool to avoid+ # blocking the event loop and allow real concurrency.+ result: EmbedResult = await asyncio.to_thread(+ self._embedder.embed, embedding_msg.message+ )# Add dense vectorif result.dense_vector:diff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.pyindex 6baeb3986..ca0e9b297 100644--- a/openviking/storage/queuefs/named_queue.py+++ b/openviking/storage/queuefs/named_queue.py@@ -198,43 +198,58 @@ class NamedQueue:msg_id = self._agfs.write(enqueue_file, data.encode("utf-8"))return msg_id if isinstance(msg_id, str) else str(msg_id)+ def _read_queue_message(self) -> Optional[Dict[str, Any]]:+ """Read and remove one message from the AGFS queue; return parsed dict or None.++ Normalises the various return types AGFSClient.read() may produce.+ """+ content = self._agfs.read(f"{self.path}/dequeue")+ if not content or content == b"{}":+ return None+ if isinstance(content, bytes):+ raw = content+ elif isinstance(content, str):+ raw = content.encode("utf-8")+ elif hasattr(content, "content") and content.content is not None:+ raw = content.content+ else:+ raw = str(content).encode("utf-8")+ return json.loads(raw.decode("utf-8"))+async def dequeue(self) -> Optional[Dict[str, Any]]:- """Get and remove message from queue (dequeue)."""+ """Get and remove message from queue, then invoke the dequeue handler."""await self._ensure_initialized()- dequeue_file = f"{self.path}/dequeue"-try:- content = self._agfs.read(dequeue_file)- if not content or content == b"{}":- return None-- # Handle different return types from AGFSClient- content_bytes = None- if isinstance(content, bytes):- content_bytes = content- elif isinstance(content, str):- content_bytes = content.encode("utf-8")- elif hasattr(content, "content"): # Response object- content_obj = content.content- if content_obj is not None:- content_bytes = content_obj- else:- content_bytes = str(content).encode("utf-8")-- if content_bytes is None:+ data = self._read_queue_message()+ if data is None:return None- data = json.loads(content_bytes.decode("utf-8"))-- # Dequeue success, mark in_progressif self._dequeue_handler:self._on_dequeue_start()data = await self._dequeue_handler.on_dequeue(data)-return dataexcept Exception as e:logger.debug(f"[NamedQueue] Dequeue failed for {self.name}: {e}")return None+ async def dequeue_raw(self) -> Optional[Dict[str, Any]]:+ """Get and remove message from queue without invoking the handler."""+ await self._ensure_initialized()+ try:+ return self._read_queue_message()+ except Exception as e:+ logger.debug(f"[NamedQueue] Dequeue raw failed for {self.name}: {e}")+ return None++ async def process_dequeued(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]:+ """Invoke the dequeue handler on already-fetched raw data.++ NOTE: caller must call _on_dequeue_start() before invoking this method+ so that in_progress is incremented atomically with the dequeue.+ """+ if self._dequeue_handler:+ return await self._dequeue_handler.on_dequeue(data)+ return data+async def peek(self) -> Optional[Dict[str, Any]]:"""Peek at head message without removing."""await self._ensure_initialized()diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 8832f8b05..7ceab4dfe 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -9,7 +9,8 @@ import asyncioimport atexitimport threadingimport time-from typing import Any, Dict, Optional, Union+import traceback+from typing import Any, Dict, Optional, Set, Unionfrom pyagfs import AGFSClient@@ -29,6 +30,7 @@ def init_queue_manager(agfs_url: str = "http://localhost:8080",timeout: int = 10,mount_point: str = "/queue",+ max_concurrent_embedding: int = 1,) -> "QueueManager":"""Initialize QueueManager singleton."""global _instance@@ -36,6 +38,7 @@ def init_queue_manager(agfs_url=agfs_url,timeout=timeout,mount_point=mount_point,+ max_concurrent_embedding=max_concurrent_embedding,)return _instance@@ -63,11 +66,13 @@ class QueueManager:agfs_url: str = "http://localhost:8080",timeout: int = 10,mount_point: str = "/queue",+ max_concurrent_embedding: int = 1,):"""Initialize QueueManager."""self._agfs_url = agfs_urlself.timeout = timeoutself.mount_point = mount_point+ self._max_concurrent_embedding = max_concurrent_embeddingself._agfs: Optional[Any] = Noneself._queues: Dict[str, NamedQueue] = {}self._started = False@@ -137,41 +142,101 @@ class QueueManager:if thread.is_alive():return+ max_concurrent = self._max_concurrent_embedding if queue.name == self.EMBEDDING else 1stop_event = threading.Event()self._queue_stop_events[queue.name] = stop_eventthread = threading.Thread(target=self._queue_worker_loop,- args=(queue, stop_event),+ args=(queue, stop_event, max_concurrent),daemon=True,)self._queue_threads[queue.name] = threadthread.start()- def _queue_worker_loop(self, queue: NamedQueue, stop_event: threading.Event) -> None:- """Worker loop for a single queue, processes items sequentially."""+ def _queue_worker_loop(+ self, queue: NamedQueue, stop_event: threading.Event, max_concurrent: int = 1+ ) -> None:+ """Worker loop for a single queue.++ When max_concurrent > 1, items are fetched and processed in parallel+ (up to max_concurrent at a time). Otherwise items are processed one by one.+ """loop = asyncio.new_event_loop()asyncio.set_event_loop(loop)try:- while not stop_event.is_set():- try:- queue_size = loop.run_until_complete(queue.size())- if queue.has_dequeue_handler() and queue_size > 0:- data = loop.run_until_complete(queue.dequeue())- if data is not None:- logger.debug(- f"[QueueManager] Dequeued message from {queue.name}: {data}"- )- else:+ if max_concurrent > 1:+ loop.run_until_complete(+ self._worker_async_concurrent(queue, stop_event, max_concurrent)+ )+ else:+ while not stop_event.is_set():+ try:+ queue_size = loop.run_until_complete(queue.size())+ if queue.has_dequeue_handler() and queue_size > 0:+ data = loop.run_until_complete(queue.dequeue())+ if data is not None:+ logger.debug(+ f"[QueueManager] Dequeued message from {queue.name}: {data}"+ )+ else:+ stop_event.wait(self._poll_interval)+ except Exception as e:+ logger.error(f"[QueueManager] Worker error for {queue.name}: {e}")+ traceback.print_exc()stop_event.wait(self._poll_interval)- except Exception as e:- logger.error(f"[QueueManager] Worker error for {queue.name}: {e}")- import traceback-- traceback.print_exc()- stop_event.wait(self._poll_interval)finally:loop.close()+ async def _worker_async_concurrent(+ self, queue: NamedQueue, stop_event: threading.Event, max_concurrent: int+ ) -> None:+ """Concurrent worker: drains the queue and processes items in parallel.++ A Semaphore caps inflight tasks at max_concurrent.+ """+ sem = asyncio.Semaphore(max_concurrent)+ active_tasks: Set[asyncio.Task] = set()++ async def process_one(data: Dict[str, Any]) -> None:+ async with sem:+ try:+ await queue.process_dequeued(data)+ except Exception as e:+ # Handler did not call report_error; decrement in_progress manually.+ queue._on_process_error(str(e), data)+ logger.error(f"[QueueManager] Concurrent worker error for {queue.name}: {e}")++ while not stop_event.is_set():+ # Prune completed tasks+ active_tasks = {t for t in active_tasks if not t.done()}++ # While capacity remains, keep draining the queue+ while len(active_tasks) < max_concurrent:+ try:+ queue_size = await queue.size()+ except Exception:+ break+ if not queue.has_dequeue_handler() or queue_size == 0:+ break+ data = await queue.dequeue_raw()+ if data is None:+ break+ # Increment before task creation to close the race window where+ # size=0 and in_progress=0 between dequeue_raw() and task execution.+ queue._on_dequeue_start()+ task = asyncio.create_task(process_one(data))+ active_tasks.add(task)+ logger.debug(+ f"[QueueManager] Dispatched concurrent task for {queue.name} "+ f"(active={len(active_tasks)})"+ )++ await asyncio.sleep(self._poll_interval)++ # Drain remaining in-flight tasks on shutdown+ if active_tasks:+ await asyncio.gather(*active_tasks, return_exceptions=True)+def stop(self) -> None:"""Stop QueueManager and release resources."""global _instancediff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex f01a42629..d47bcf40e 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -111,6 +111,7 @@ def _enable_viking_fs_recorder(viking_fs: "VikingFS") -> None:recorder = get_recorder()if not recorder.enabled:from openviking.eval.recorder import init_recorder+init_recorder(enabled=True)global _instance@@ -964,11 +965,24 @@ class VikingFS:async def read_file(self,uri: str,+ offset: int = 0,+ limit: int = -1,) -> str:- """Read single file."""+ """Read single file, optionally sliced by line range.++ Args:+ uri: Viking URI+ offset: Starting line number (0-indexed). Default 0.+ limit: Number of lines to read. -1 means read to end. Default -1.+ """path = self._uri_to_path(uri)content = self.agfs.read(path)- return self._handle_agfs_content(content)+ text = self._handle_agfs_content(content)+ if offset == 0 and limit == -1:+ return text+ lines = text.splitlines(keepends=True)+ sliced = lines[offset:] if limit == -1 else lines[offset : offset + limit]+ return "".join(sliced)async def read_file_bytes(self,diff --git a/openviking/sync_client.py b/openviking/sync_client.pyindex 8dc1f60b2..936328bb9 100644--- a/openviking/sync_client.py+++ b/openviking/sync_client.py@@ -128,9 +128,9 @@ class SyncOpenViking:"""Read L1 overview"""return run_async(self._async_client.overview(uri))- def read(self, uri: str) -> str:+ def read(self, uri: str, offset: int = 0, limit: int = -1) -> str:"""Read file"""- return run_async(self._async_client.read(uri))+ return run_async(self._async_client.read(uri, offset=offset, limit=limit))def ls(self, uri: str, **kwargs) -> List[Any]:"""diff --git a/openviking_cli/cli/commands/content.py b/openviking_cli/cli/commands/content.pyindex d20462e89..613eefa45 100644--- a/openviking_cli/cli/commands/content.py+++ b/openviking_cli/cli/commands/content.py@@ -14,9 +14,11 @@ def register(app: typer.Typer) -> None:def read_command(ctx: typer.Context,uri: str = typer.Argument(..., help="Viking URI"),+ offset: int = typer.Option(0, "--offset", "-s", help="Starting line number (0-indexed)"),+ limit: int = typer.Option(-1, "--limit", "-n", help="Number of lines to read (-1 = all)"),) -> None:"""Read full file content (L2)."""- run(ctx, lambda client: client.read(uri))+ run(ctx, lambda client: client.read(uri, offset=offset, limit=limit))@app.command("abstract")def abstract_command(diff --git a/openviking_cli/client/base.py b/openviking_cli/client/base.pyindex 7882b5852..4141ba681 100644--- a/openviking_cli/client/base.py+++ b/openviking_cli/client/base.py@@ -108,8 +108,14 @@ class BaseClient(ABC):# ============= Content Reading =============@abstractmethod- async def read(self, uri: str) -> str:- """Read file content (L2)."""+ async def read(self, uri: str, offset: int = 0, limit: int = -1) -> str:+ """Read file content (L2).++ Args:+ uri: Viking URI+ offset: Starting line number (0-indexed). Default 0.+ limit: Number of lines to read. -1 means read to end. Default -1.+ """...@abstractmethoddiff --git a/openviking_cli/client/http.py b/openviking_cli/client/http.pyindex 859bdd95b..13e44b31d 100644--- a/openviking_cli/client/http.py+++ b/openviking_cli/client/http.py@@ -417,12 +417,12 @@ class AsyncHTTPClient(BaseClient):# ============= Content Reading =============- async def read(self, uri: str) -> str:+ async def read(self, uri: str, offset: int = 0, limit: int = -1) -> str:"""Read file content."""uri = VikingURI.normalize(uri)response = await self._http.get("/api/v1/content/read",- params={"uri": uri},+ params={"uri": uri, "offset": offset, "limit": limit},)return self._handle_response(response)diff --git a/openviking_cli/client/sync_http.py b/openviking_cli/client/sync_http.pyindex 7970ee267..40ea2e120 100644--- a/openviking_cli/client/sync_http.py+++ b/openviking_cli/client/sync_http.py@@ -213,9 +213,9 @@ class SyncHTTPClient:# ============= Content =============- def read(self, uri: str) -> str:+ def read(self, uri: str, offset: int = 0, limit: int = -1) -> str:"""Read file content."""- return run_async(self._async_client.read(uri))+ return run_async(self._async_client.read(uri, offset=offset, limit=limit))def abstract(self, uri: str) -> str:"""Read L0 abstract."""diff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.pyindex ba9aab96e..5184ddab6 100644--- a/openviking_cli/utils/config/embedding_config.py+++ b/openviking_cli/utils/config/embedding_config.py@@ -15,7 +15,8 @@ class EmbeddingModelConfig(BaseModel):batch_size: int = Field(default=32, description="Batch size for embedding generation")input: str = Field(default="multimodal", description="Input type: 'text' or 'multimodal'")provider: Optional[str] = Field(- default="volcengine", description="Provider type: 'openai', 'volcengine', 'vikingdb', 'jina'"+ default="volcengine",+ description="Provider type: 'openai', 'volcengine', 'vikingdb', 'jina'",)backend: Optional[str] = Field(default="volcengine",@@ -103,6 +104,10 @@ class EmbeddingConfig(BaseModel):sparse: Optional[EmbeddingModelConfig] = Field(default=None)hybrid: Optional[EmbeddingModelConfig] = Field(default=None)+ max_concurrent: int = Field(+ default=1, description="Maximum number of concurrent embedding requests"+ )+model_config = {"extra": "forbid"}@model_validator(mode="after")diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.pyindex 4696b9103..9504b4368 100644--- a/tests/test_edge_cases.py+++ b/tests/test_edge_cases.py@@ -14,25 +14,21 @@ import asyncioimport jsonimport osimport sys-import tempfile-import threadingimport unicodedata-from concurrent.futures import ThreadPoolExecutor, as_completedfrom pathlib import Pathfrom typing import Dict, List, Optional-from unittest.mock import AsyncMock, MagicMock, patch+from unittest.mock import AsyncMock, MagicMockimport pytestsys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))-from openviking.parse.parsers.upload_utils import (+from openviking.parse.parsers.upload_utils import ( # noqa: I001_sanitize_rel_path,detect_and_convert_encoding,is_text_file,upload_directory,)-from openviking.storage.viking_fs import VikingFSfrom openviking.storage.vikingdb_interface import VikingDBInterfacefrom openviking.utils.compression import CompressManagerfrom openviking_cli.utils.uri import VikingURI@@ -40,24 +36,26 @@ from openviking_cli.utils.uri import VikingURIclass MockVikingDB(VikingDBInterface):"""Mock vector database for testing."""-+def __init__(self):self.collections: Dict[str, Dict] = {}self.data: Dict[str, List[Dict]] = {}self.deleted_ids: set = set()-+async def create_collection(self, name: str, schema: Dict) -> bool:if name in self.collections:return Falseself.collections[name] = schemaself.data[name] = []return True-- async def search_by_id(self, collection: str, doc_id: str, candidates: Optional[List[str]] = None) -> Optional[Dict]:++ async def search_by_id(+ self, collection: str, doc_id: str, candidates: Optional[List[str]] = None+ ) -> Optional[Dict]:"""Search for document by ID with optional candidate filtering."""if collection not in self.data:return None-+if candidates is None:# Search all documentsfor doc in self.data[collection]:@@ -68,17 +66,21 @@ class MockVikingDB(VikingDBInterface):if not candidates: # Empty candidate listreturn Nonefor doc in self.data[collection]:- if doc.get("id") == doc_id and doc_id in candidates and doc_id not in self.deleted_ids:+ if (+ doc.get("id") == doc_id+ and doc_id in candidates+ and doc_id not in self.deleted_ids+ ):return doc-+return None-+async def insert(self, collection: str, data: List[Dict]) -> bool:if collection not in self.data:return Falseself.data[collection].extend(data)return True-+async def delete(self, collection: str, doc_id: str) -> bool:self.deleted_ids.add(doc_id)return True@@ -86,62 +88,62 @@ class MockVikingDB(VikingDBInterface):class TestLongFilenames:"""Test handling of very long filenames and path components."""-+def test_filename_exactly_255_bytes(self):"""Test filename with exactly 255 bytes (filesystem limit boundary)."""# Create a filename that's exactly 255 bytes in UTF-8base_name = "a" * 251 # 251 + ".txt" = 255 bytesfilename = base_name + ".txt"-- assert len(filename.encode('utf-8')) == 255-++ assert len(filename.encode("utf-8")) == 255+# Test sanitization doesn't break at exact boundarysanitized = _sanitize_rel_path(filename)assert sanitized is not Noneassert len(sanitized) > 0-+def test_filename_256_bytes_boundary(self):"""Test filename with 256 bytes (just over filesystem limit)."""# Create filename that's exactly 256 bytes - should be truncated- base_name = "b" * 252 # 252 + ".txt" = 256 bytes+ base_name = "b" * 252 # 252 + ".txt" = 256 bytesfilename = base_name + ".txt"-- assert len(filename.encode('utf-8')) == 256-++ assert len(filename.encode("utf-8")) == 256+sanitized = _sanitize_rel_path(filename)# Should be handled gracefully (truncated or rejected)assert sanitized is not None-+def test_very_long_filename_with_cjk(self):"""Test extremely long filename with CJK characters (3 bytes per char in UTF-8)."""# Each CJK character is 3 bytes in UTF-8cjk_chars = "测试文件名" * 30 # ~450 bytesfilename = f"{cjk_chars}.py"-- assert len(filename.encode('utf-8')) > 400-++ assert len(filename.encode("utf-8")) > 400+sanitized = _sanitize_rel_path(filename)assert sanitized is not None# Should handle or truncate appropriately-+def test_filename_only_special_characters(self):"""Test filename composed entirely of special characters."""special_filename = "!@#$%^&*()_+-={}[]|\\:;\"'<>,.?/~`" + ".txt"-+sanitized = _sanitize_rel_path(special_filename)# Should sanitize dangerous characters while preserving valid onesassert sanitized is not Noneassert ".txt" in sanitized # Extension should be preserved-+def test_filename_with_path_traversal_attempts(self):"""Test filename containing path traversal sequences."""dangerous_filenames = ["../../../etc/passwd","..\\..\\windows\\system32\\config","file/../../../secret.txt",- "normal_file_../../../dangerous.py"+ "normal_file_../../../dangerous.py",]-+for filename in dangerous_filenames:sanitized = _sanitize_rel_path(filename)# Should not contain path traversal sequences@@ -151,241 +153,237 @@ class TestLongFilenames:class TestSearchByIdEdgeCases:"""Test search_by_id with various edge cases and None conditions."""-+@pytest.mark.asyncioasync def test_search_nonexistent_id(self):"""Test searching for an ID that doesn't exist."""mock_db = MockVikingDB()await mock_db.create_collection("test", {})-+result = await mock_db.search_by_id("test", "nonexistent_id")assert result is None-+@pytest.mark.asyncioasync def test_search_after_delete(self):"""Test searching for an ID after it has been deleted."""mock_db = MockVikingDB()await mock_db.create_collection("test", {})-+# Insert documentawait mock_db.insert("test", [{"id": "doc1", "content": "test"}])-+# Verify it existsresult = await mock_db.search_by_id("test", "doc1")assert result is not None-+# Delete itawait mock_db.delete("test", "doc1")-+# Search should return Noneresult = await mock_db.search_by_id("test", "doc1")assert result is None-+@pytest.mark.asyncioasync def test_search_with_empty_candidates(self):"""Test search_by_id with empty candidate list."""mock_db = MockVikingDB()await mock_db.create_collection("test", {})-+# Insert documentawait mock_db.insert("test", [{"id": "doc1", "content": "test"}])-+# Search with empty candidates should return Noneresult = await mock_db.search_by_id("test", "doc1", candidates=[])assert result is None-+@pytest.mark.asyncioasync def test_search_with_none_candidates(self):"""Test search_by_id with None candidates (should search all)."""mock_db = MockVikingDB()await mock_db.create_collection("test", {})-+# Insert documentawait mock_db.insert("test", [{"id": "doc1", "content": "test"}])-+# Search with None candidates should find documentresult = await mock_db.search_by_id("test", "doc1", candidates=None)assert result is not Noneassert result["id"] == "doc1"-+@pytest.mark.asyncioasync def test_search_nonexistent_collection(self):"""Test searching in a collection that doesn't exist."""mock_db = MockVikingDB()-+result = await mock_db.search_by_id("nonexistent", "doc1")assert result is Noneclass TestDuplicateFilenameHandling:"""Test duplicate filename handling and case sensitivity."""-+@pytest.mark.asyncioasync def test_upload_same_file_multiple_times(self, tmp_path):"""Test uploading the same file 10 times - should handle duplicates gracefully."""# Create test filetest_file = tmp_path / "duplicate_test.txt"test_file.write_text("This is a test file for duplicate testing.")-+# Mock VikingFSmock_fs = MagicMock()mock_fs.write_file_bytes = AsyncMock()mock_fs.mkdir = AsyncMock()-+# Upload the same file 10 times- for i in range(10):+ for _ in range(10):await upload_text_files([str(test_file)], "viking://test/", mock_fs)-+# Should handle duplicates without crashingassert mock_fs.write_file_bytes.call_count == 10-+def test_case_sensitivity_filenames(self):"""Test filenames that differ only in case."""- filenames = [- "TestFile.txt",- "testfile.txt",- "TESTFILE.TXT",- "TestFile.TXT"- ]-+ filenames = ["TestFile.txt", "testfile.txt", "TESTFILE.TXT", "TestFile.TXT"]+sanitized_names = [_sanitize_rel_path(name) for name in filenames]-+# All should be valid but may be treated differently on case-insensitive systemsfor name in sanitized_names:assert name is not Noneassert len(name) > 0-+def test_unicode_normalization_differences(self):"""Test filenames with different Unicode normalizations (NFC vs NFD)."""# Same logical character represented differentlyfilename_nfc = "café.txt" # NFC: é is a single codepointfilename_nfd = "cafe\u0301.txt" # NFD: e + combining acute accent-+# These look the same but have different byte representationsassert filename_nfc != filename_nfd- assert unicodedata.normalize('NFC', filename_nfd) == filename_nfc-+ assert unicodedata.normalize("NFC", filename_nfd) == filename_nfc+sanitized_nfc = _sanitize_rel_path(filename_nfc)sanitized_nfd = _sanitize_rel_path(filename_nfd)-+assert sanitized_nfc is not Noneassert sanitized_nfd is not Noneclass TestCompressionEdgeCases:"""Test compression with various edge cases."""-+def test_compress_empty_string(self):"""Test compressing empty string."""compressor = CompressManager()-+empty_text = ""compressed = compressor.compress_text(empty_text)-+assert compressed is not Noneassert len(compressed) >= 0 # May be empty or minimal header-+def test_compress_emoji_only(self):"""Test compressing string containing only emoji."""compressor = CompressManager()-+emoji_text = "😀😃😄😁😆😅😂🤣😊😇🙂🙃😉😌😍🥰😘😗😙😚"compressed = compressor.compress_text(emoji_text)-+assert compressed is not None# Emoji compression ratio might be poor due to lack of repetitionassert len(compressed) > 0-+def test_compress_large_text(self):"""Test compressing 1MB of text."""compressor = CompressManager()-+# Generate ~1MB of text with patterns (should compress well)large_text = "This is a repeating pattern for compression testing. " * 20000-- assert len(large_text.encode('utf-8')) > 1_000_000-++ assert len(large_text.encode("utf-8")) > 1_000_000+compressed = compressor.compress_text(large_text)-+assert compressed is not None# Should achieve significant compression due to repetition- compression_ratio = len(compressed) / len(large_text.encode('utf-8'))+ compression_ratio = len(compressed) / len(large_text.encode("utf-8"))assert compression_ratio < 0.1 # Should compress to less than 10%-+def test_compress_binary_like_data(self):"""Test compressing text that looks like binary data."""compressor = CompressManager()-+# Pseudo-random looking text (poor compression expected)import hashlib+binary_like = ""for i in range(1000):binary_like += hashlib.md5(f"seed{i}".encode()).hexdigest()-+compressed = compressor.compress_text(binary_like)assert compressed is not None# Compression ratio should be poor for random-looking data- compression_ratio = len(compressed) / len(binary_like.encode('utf-8'))+ compression_ratio = len(compressed) / len(binary_like.encode("utf-8"))assert compression_ratio > 0.8 # Should not compress muchclass TestConcurrentOperations:"""Test concurrent operations for race conditions and thread safety."""-+@pytest.mark.asyncioasync def test_concurrent_writes(self):"""Test 20 parallel write operations."""mock_fs = MagicMock()mock_fs.write_file_bytes = AsyncMock()mock_fs.mkdir = AsyncMock()-+# Create 20 concurrent write tasksasync def write_task(i):content = f"Content for file {i}"uri = f"viking://concurrent/file_{i}.txt"- await mock_fs.write_file_bytes(uri, content.encode('utf-8'))-+ await mock_fs.write_file_bytes(uri, content.encode("utf-8"))+tasks = [write_task(i) for i in range(20)]-+# Execute all tasks concurrentlyawait asyncio.gather(*tasks)-+# Verify all writes were attemptedassert mock_fs.write_file_bytes.call_count == 20-- @pytest.mark.asyncio++ @pytest.mark.asyncioasync def test_concurrent_search_while_writing(self):"""Test 10 parallel searches while writing."""mock_db = MockVikingDB()await mock_db.create_collection("concurrent", {})-+# Insert initial datafor i in range(5):await mock_db.insert("concurrent", [{"id": f"doc{i}", "content": f"content{i}"}])-+async def search_task():return await mock_db.search_by_id("concurrent", "doc1")-+async def write_task():return await mock_db.insert("concurrent", [{"id": "new_doc", "content": "new_content"}])-+# Mix of search and write operationstasks = []tasks.extend([search_task() for _ in range(10)])tasks.extend([write_task() for _ in range(5)])-+results = await asyncio.gather(*tasks, return_exceptions=True)-+# No tasks should have raised exceptionsfor result in results:assert not isinstance(result, Exception)-+@pytest.mark.asyncioasync def test_rapid_create_delete_cycles(self):"""Test rapid create/delete cycles for race conditions."""mock_db = MockVikingDB()await mock_db.create_collection("rapid", {})-+async def create_delete_cycle(doc_id):# Create documentawait mock_db.insert("rapid", [{"id": doc_id, "content": "temp"}])@@ -396,86 +394,86 @@ class TestConcurrentOperations:# Search again (should be None)deleted_result = await mock_db.search_by_id("rapid", doc_id)return result, deleted_result-+# Run 10 rapid create/delete cyclestasks = [create_delete_cycle(f"rapid_doc_{i}") for i in range(10)]results = await asyncio.gather(*tasks)-+# Verify results are consistentfor found, deleted in results:assert found is not None # Should find before delete- assert deleted is None # Should not find after delete+ assert deleted is None # Should not find after deleteclass TestUnicodeEdgeCases:"""Test Unicode edge cases and special character handling."""-+def test_zero_width_characters(self):"""Test filenames containing zero-width characters."""# Zero-width characters that might cause issuesfilename = "test\u200b\u200c\u200d\ufefffile.txt" # ZWSP, ZWNJ, ZWJ, BOM-+sanitized = _sanitize_rel_path(filename)assert sanitized is not None-+# Zero-width characters should ideally be strippedassert "\u200b" not in sanitized or len(sanitized) > 0-+def test_rtl_text_filenames(self):"""Test right-to-left text in filenames."""# Arabic/Hebrew filenamertl_filename = "ملف_اختبار.txt" # Arabic for "test file"-+sanitized = _sanitize_rel_path(rtl_filename)assert sanitized is not Noneassert len(sanitized) > 0-+# Should preserve RTL charactersassert "ملف" in sanitized-+def test_combining_characters(self):"""Test filenames with combining characters."""# Base character + multiple combining marksfilename = "e\u0301\u0302\u0303\u0304.txt" # e + acute + circumflex + tilde + macron-+sanitized = _sanitize_rel_path(filename)assert sanitized is not Noneassert len(sanitized) > 0-+def test_surrogate_pairs(self):"""Test filenames with surrogate pairs (emoji, etc)."""# Emoji that require surrogate pairs in UTF-16filename = "test🏴🧑💻👨👩👧👦.txt" # Flag, person, family-+sanitized = _sanitize_rel_path(filename)- assert sanitized is not None+ assert sanitized is not Noneassert len(sanitized) > 0-+# Should handle complex emoji sequencesclass TestSecurityEdgeCases:"""Test security-related edge cases."""-+def test_null_bytes_in_content(self):"""Test handling of null bytes in file content."""content_with_nulls = "Hello\x00World\x00Test"-+# Should handle gracefully without crashing- encoding_result = detect_and_convert_encoding(content_with_nulls.encode('utf-8'))+ encoding_result = detect_and_convert_encoding(content_with_nulls.encode("utf-8"))assert encoding_result is not None-+def test_deeply_nested_json(self):"""Test handling of very deeply nested JSON structures."""# Create deeply nested JSON (potential DoS via recursion)- nested_json = "{"+ nested_json = "{"for _ in range(1000):nested_json += '"key": {'nested_json += '"value": "deep"'for _ in range(1000):nested_json += "}"nested_json += "}"-+# Should handle without stack overflowtry:parsed = json.loads(nested_json)@@ -483,7 +481,7 @@ class TestSecurityEdgeCases:except (json.JSONDecodeError, RecursionError):# Either parsing fails gracefully or recursion is limitedpass-+def test_malformed_uri_handling(self):"""Test handling of malformed URIs."""malformed_uris = [@@ -493,7 +491,7 @@ class TestSecurityEdgeCases:"viking://path with spaces", # Unescaped spaces"viking://../../../etc/passwd", # Path traversal]-+for uri in malformed_uris:try:viking_uri = VikingURI(uri)@@ -506,7 +504,7 @@ class TestSecurityEdgeCases:class TestBoundaryConditions:"""Test various boundary conditions and limits."""-+def test_is_text_file_edge_cases(self):"""Test is_text_file with edge case filenames."""edge_cases = [@@ -520,7 +518,7 @@ class TestBoundaryConditions:"file.TXT", # Uppercase extension"FILE.txt", # Mixed case]-+for filename in edge_cases:# Should not crashtry:@@ -529,34 +527,47 @@ class TestBoundaryConditions:except Exception:# May raise exception for invalid filenames - that's OKpass-- def test_directory_upload_with_circular_symlinks(self, tmp_path):++ @pytest.mark.asyncio+ async def test_directory_upload_with_circular_symlinks(self, tmp_path):"""Test directory upload with circular symbolic links."""- if os.name == 'nt': # Skip on Windows due to symlink permissions+ if os.name == "nt": # Skip on Windows due to symlink permissionspytest.skip("Symlink test skipped on Windows")-+# Create directoriesdir_a = tmp_path / "dir_a"dir_b = tmp_path / "dir_b"dir_a.mkdir()dir_b.mkdir()-+# Create circular symlinks(dir_a / "link_to_b").symlink_to(dir_b)(dir_b / "link_to_a").symlink_to(dir_a)-+# Add a regular file(dir_a / "test.txt").write_text("test content")-- mock_fs = MagicMock()- mock_fs.write_file_bytes = AsyncMock()- mock_fs.mkdir = AsyncMock()-++ class FakeAGFS:+ def mkdir(self, path: str) -> None:+ pass++ def write(self, path: str, content: bytes) -> None:+ pass++ class MockFS:+ agfs = FakeAGFS()++ def _uri_to_path(self, uri: str) -> str:+ return uri++ async def mkdir(self, uri: str, exist_ok: bool = False) -> None:+ pass+# Should handle circular links without infinite recursiontry:- result = await upload_directory(str(tmp_path), "viking://test/", mock_fs)+ result = await upload_directory(tmp_path, "viking://test/", MockFS())# Should complete without hanging- assert result is None or result is not None # Just checking it returns+ assert result is None or result is not Noneexcept Exception as e:# Acceptable to raise exception for circular linksassert "recursion" in str(e).lower() or "circular" in str(e).lower()@@ -570,4 +581,4 @@ async def upload_text_files(file_paths: List[str], target_uri: str, viking_fs):if path.exists() and path.is_file():content = path.read_bytes()uri = f"{target_uri.rstrip('/')}/{path.name}"- await viking_fs.write_file_bytes(uri, content)\ No newline at end of file+ await viking_fs.write_file_bytes(uri, content)diff --git a/tests/test_upload_utils.py b/tests/test_upload_utils.pyindex f334a5839..6a47803d8 100644--- a/tests/test_upload_utils.py+++ b/tests/test_upload_utils.py@@ -22,12 +22,31 @@ from openviking.parse.parsers.upload_utils import (# ---------------------------------------------------------------------------+class FakeAGFS:+ """Minimal AGFS mock that stores files and directories by path key."""++ def __init__(self, storage: Dict[str, bytes]) -> None:+ self._storage = storage+ self.dirs: List[str] = []++ def mkdir(self, path: str) -> None:+ self.dirs.append(path)++ def write(self, path: str, content: bytes) -> None:+ self._storage[path] = content++class FakeVikingFS:"""Minimal VikingFS mock for testing upload functions."""def __init__(self) -> None:self.files: Dict[str, bytes] = {}self.dirs: List[str] = []+ self.agfs = FakeAGFS(self.files)++ def _uri_to_path(self, uri: str) -> str:+ # Simplified: use the URI itself as the storage key so test assertions work.+ return uriasync def write_file_bytes(self, uri: str, content: bytes) -> None:self.files[uri] = content@@ -328,7 +347,9 @@ class TestUploadDirectory:@pytest.mark.asyncioasync def test_creates_root_dir(self, tmp_dir: Path, viking_fs: FakeVikingFS) -> None:await upload_directory(tmp_dir, "viking://temp/root", viking_fs)- assert "viking://temp/root" in viking_fs.dirs+ # _mkdir_with_parents strips leading slash then re-adds it, so the stored agfs+ # path is the _uri_to_path() result with a "/" prefix.+ assert any("temp/root" in d for d in viking_fs.agfs.dirs)@pytest.mark.asyncioasync def test_custom_ignore_dirs(self, tmp_dir: Path, viking_fs: FakeVikingFS) -> None:@@ -477,10 +498,19 @@ class TestUploadTextFilesEdgeCases:class TestUploadDirectoryEdgeCases:@pytest.mark.asyncioasync def test_write_failure_produces_warning(self, tmp_path: Path) -> None:- class FailingWriteFS:- async def write_file_bytes(self, uri: str, content: bytes) -> None:+ class FailingAGFS:+ def mkdir(self, path: str) -> None:+ pass++ def write(self, path: str, content: bytes) -> None:raise IOError("write error")+ class FailingWriteFS:+ agfs = FailingAGFS()++ def _uri_to_path(self, uri: str) -> str:+ return uri+async def mkdir(self, uri: str, exist_ok: bool = False) -> None:pass