Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
Code Repository Parser.
Handles git repositories and zip archives of codebases.
Implements V5.0 asynchronous architecture:
- Physical move (Clone -> Temp VikingFS)
- No LLM generation in parser phase
"""
import asyncio
import os
import shutil
import stat
import time
import urllib.request
import zipfile
from pathlib import Path, PurePosixPath
from typing import Any, List, Optional, Set, Tuple, Union
from urllib.parse import unquote, urlparse
from openviking.parse.base import (
NodeType,
ParseResult,
ResourceNode,
create_parse_result,
)
from openviking.parse.parsers.base_parser import BaseParser
from openviking.parse.parsers.constants import (
CODE_EXTENSIONS,
DOCUMENTATION_EXTENSIONS,
FILE_TYPE_CODE,
FILE_TYPE_DOCUMENTATION,
FILE_TYPE_OTHER,
IGNORE_DIRS,
IGNORE_EXTENSIONS,
)
from openviking.parse.parsers.upload_utils import upload_directory
from openviking.utils import is_github_url, parse_code_hosting_url
from openviking.utils.code_hosting_utils import _domain_matches
from openviking_cli.utils.config import get_openviking_config
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
class CodeRepositoryParser(BaseParser):
"""
Parser for code repositories (Git/Zip).
Features:
- Shallow clone for Git repositories
- Automatic filtering based on .gitignore and of non-code directories (.git, node_modules, etc.)
- Direct mapping to VikingFS temp directory
- Preserves directory structure without chunking
代码仓库入库处理流程
输入: https://github.com/markwhen/gogetxueqiu
↓
[GitAccessor] → LocalResource
- path: /tmp/.../extracted/repo
- source_type: SourceType.GIT
- original_source: "https://github.com/markwhen/gogetxueqiu"
- meta: {repo_name: "markwhen/gogetxueqiu", ...}
↓
[media_processor] → 所有目录都用 DirectoryParser!← 简化了!
↓
[DirectoryParser.parse()]
├─→ 检测到 (path/.git).exists() ← 新增!
├─→ 收集 git 元数据
└─→ 委托给 CodeRepositoryParser.parse()
↓
[CodeRepositoryParser.parse()]
- source_path = original_source (https://github.com/...)
↓
[TreeBuilder.finalize_from_temp()]
- 从 source_path 解析出 "markwhen/gogetxueqiu"
- root_uri = "viking://resources/markwhen/gogetxueqiu"
"""
# Class constants imported from constants.py
IGNORE_DIRS = IGNORE_DIRS
IGNORE_EXTENSIONS = IGNORE_EXTENSIONS
@property
def supported_extensions(self) -> List[str]:
# This parser is primarily invoked by URLTypeDetector, not by file extension
return [".git", ".zip"]
def _detect_file_type(self, file_path: Path) -> str:
"""
Detect file type based on extension for potential metadata tagging.
Returns:
"code" for programming language files
"documentation" for documentation files (md, txt, rst, etc.)
"other" for other text files
"binary" for binary files (already filtered by IGNORE_EXTENSIONS)
"""
extension = file_path.suffix.lower()
if extension in CODE_EXTENSIONS:
return FILE_TYPE_CODE
elif extension in DOCUMENTATION_EXTENSIONS:
return FILE_TYPE_DOCUMENTATION
else:
# For other text files not in the lists
return FILE_TYPE_OTHER
async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult:
"""
Parse code repository (only accepts local directories, as network fetching is done in Accessor layer).
Args:
source: Local directory path (already fetched by DataAccessor)
instruction: Processing instruction (unused in parser phase)
**kwargs: Additional arguments
_source_meta: Metadata from DataAccessor (must contain repo_name, repo_ref, repo_commit)
original_source: Original URL (for repo name extraction if needed)
Returns:
ParseResult with temp_dir_path pointing to the uploaded content
"""
start_time = time.time()
source_path = Path(source)
# Always work through a ParseOutputStore. Local mode threads one in via
# kwargs; otherwise default to the AGFS-backed store so the parser has a
# single, backend-agnostic write path (no viking_fs/store branching).
output_store = kwargs.get("parse_output_store")
if output_store is None:
from openviking.parse.output import AgfsParseOutputStore
output_store = AgfsParseOutputStore(viking_fs=self._get_viking_fs())
artifact_ref = None
# Check if source is already a local directory (should always be true)
if not source_path.is_dir():
raise ValueError(
f"CodeRepositoryParser only accepts local directories. "
f"Source type: {type(source)}, value: {source}"
)
logger.info(f"[CodeRepositoryParser] Parsing code repository: {source_path}")
try:
# Get metadata from DataAccessor
# _source_meta comes from GitAccessor and contains:
# - repo_name: in "org/repo" format (e.g. "volcengine/OpenViking")
# - repo_ref: branch name if specified
# - repo_commit: commit hash if specified
source_meta = kwargs.get("_source_meta", {})
repo_name = source_meta.get("repo_name", "repository")
branch = source_meta.get("repo_ref")
commit = source_meta.get("repo_commit")
# If repo_name is still default, try to extract from original source
# original_source is the full code-hosting URL that the user provided
# (e.g. "https://github.com/volcengine/OpenViking")
if repo_name == "repository":
original_source = kwargs.get("original_source") or source_meta.get(
"original_source"
)
if original_source:
parsed_org_repo = parse_code_hosting_url(original_source)
if parsed_org_repo:
repo_name = parsed_org_repo
local_dir = source_path
# 3. Allocate the artifact root through the store. Local mode writes
# to a shared local dir; AGFS mode writes to viking://temp. Either way
# the layout is <root>/repository/... and the parser code is identical.
artifact_ref = await output_store.create_artifact(root_type="dir")
temp_viking_uri = artifact_ref.root
target_root_uri = f"{temp_viking_uri}/repository"
logger.debug(f"Uploading code repository artifacts to: {target_root_uri}")
# 4. Upload to the artifact store (filtering on the fly)
file_count, upload_failures = await self._upload_directory(
local_dir,
output_store=output_store,
artifact_ref=artifact_ref,
ignore_dirs=kwargs.get("ignore_dirs"),
include=kwargs.get("include"),
exclude=kwargs.get("exclude"),
)
if upload_failures:
detail = "; ".join(upload_failures[:5])
if len(upload_failures) > 5:
detail += f"; ... {len(upload_failures) - 5} more"
raise RuntimeError(
f"Failed to upload {len(upload_failures)} repository file(s): {detail}"
)
logger.info(f"Uploaded {file_count} files to {target_root_uri}")
# 5. Create result
# Root node is just a placeholder, TreeBuilder relies on temp_dir_path
root = ResourceNode(
type=NodeType.ROOT,
content_path=None,
meta={"name": repo_name, "type": "repository"},
)
# Use original URL as source_path instead of local temp dir for proper org/repo parsing in TreeBuilder
# source_path is CRITICAL:
# 1. TreeBuilder uses source_path to parse org/repo via parse_code_hosting_url()
# 2. If source_path is a local path (like /tmp/.../OpenViking), parsing fails
# 3. If source_path is the original code-hosting URL,
# TreeBuilder can correctly extract "volcengine/OpenViking"
#
# Priority order:
# 1. First check kwargs["original_source"] - this is set by media_processor
# 2. Then check source_meta (rarely has it)
# 3. Fall back to local path only as last resort
original_source = kwargs.get("original_source") or source_meta.get("original_source")
result = create_parse_result(
root=root,
source_path=original_source or str(source),
source_format="repository",
parser_name="CodeRepositoryParser",
parse_time=time.time() - start_time,
)
result.temp_dir_path = temp_viking_uri # Points to parent of repo_name
result.artifact_ref = artifact_ref
result.meta["file_count"] = file_count
result.meta["repo_name"] = repo_name
if branch:
result.meta["repo_ref"] = branch
if commit:
result.meta["repo_commit"] = commit
resolved_head_sha = source_meta.get("resolved_head_sha")
if resolved_head_sha:
result.meta["resolved_head_sha"] = resolved_head_sha
return result
except Exception as e:
logger.error(f"Failed to parse repository {source}: {e}", exc_info=True)
if artifact_ref is not None:
try:
await output_store.cleanup(artifact_ref)
except Exception as cleanup_error:
logger.warning(
"Failed to clean repository parse artifact %s: %s",
artifact_ref.root,
cleanup_error,
)
# Use original URL for error case as well - still important for TreeBuilder
# Even on failure, we want TreeBuilder to potentially get org/repo from the URL
original_source = kwargs.get("original_source") or source_meta.get("original_source")
return create_parse_result(
root=ResourceNode(type=NodeType.ROOT, content_path=None),
source_path=original_source or str(source),
source_format="repository",
parser_name="CodeRepositoryParser",
parse_time=time.time() - start_time,
warnings=[f"Failed to parse repository: {str(e)}"],
)
async def parse_content(
self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs
) -> ParseResult:
"""Not supported for repositories."""
raise NotImplementedError("CodeRepositoryParser does not support parse_content")
def _parse_repo_source(self, source: str, **kwargs) -> Tuple[str, Optional[str], Optional[str]]:
branch = kwargs.get("branch") or kwargs.get("ref")
commit = kwargs.get("commit")
repo_url = source
if source.startswith(("http://", "https://", "git://", "ssh://")):
parsed = urlparse(source)
repo_url = parsed._replace(query="", fragment="").geturl()
if commit is None or branch is None:
branch, commit = self._extract_ref_from_url(parsed, branch, commit)
repo_url = self._normalize_repo_url(repo_url)
return repo_url, branch, commit
def _extract_ref_from_url(
self,
parsed: Any,
branch: Optional[str],
commit: Optional[str],
) -> Tuple[Optional[str], Optional[str]]:
if parsed.path:
path_branch, path_commit = self._parse_ref_from_path(parsed.path)
commit = path_commit or commit
# If commit is present in path, ignore branch entirely
if commit:
branch = None
else:
branch = branch or path_branch
return branch, commit
def _parse_ref_from_path(self, path: str) -> Tuple[Optional[str], Optional[str]]:
parts = [p for p in path.split("/") if p]
branch = None
commit = None
if "commit" in parts:
idx = parts.index("commit")
if idx + 1 < len(parts):
commit = parts[idx + 1]
if "tree" in parts:
idx = parts.index("tree")
if idx + 1 < len(parts):
ref = unquote(parts[idx + 1])
if self._looks_like_sha(ref):
commit = ref
else:
branch = ref
return branch, commit
@staticmethod
def _looks_like_sha(ref: str) -> bool:
"""Return True if ref looks like a git commit SHA (7-40 hex chars)."""
return 7 <= len(ref) <= 40 and all(c in "0123456789abcdefABCDEF" for c in ref)
def _normalize_repo_url(self, url: str) -> str:
if url.startswith(("http://", "https://", "git://", "ssh://")):
parsed = urlparse(url)
path_parts = [p for p in parsed.path.split("/") if p]
base_parts = path_parts
git_index = next((i for i, p in enumerate(path_parts) if p.endswith(".git")), None)
if git_index is not None:
base_parts = path_parts[: git_index + 1]
config = get_openviking_config()
if _domain_matches(parsed, config.code.github_domains + config.code.gitlab_domains):
base_parts = path_parts[:2]
base_path = "/" + "/".join(base_parts)
return parsed._replace(path=base_path, query="", fragment="").geturl()
return url
def _get_repo_name(self, url: str) -> str:
"""Get repository name with organization for configured code-hosting URLs.
For https://github.com/volcengine/OpenViking, returns "volcengine/OpenViking"
For other URLs, falls back to just the repo name.
"""
# First try to parse as code hosting URL
parsed_org_repo = parse_code_hosting_url(url)
if parsed_org_repo:
return parsed_org_repo
# Fallback for other URLs
name_source = url
if url.startswith(("http://", "https://", "git://", "ssh://")):
name_source = urlparse(url).path.rstrip("/")
elif ":" in url and not url.startswith("file://"):
name_source = url.split(":", 1)[1]
# Fallback for URLs outside the configured code-hosting domains
name = name_source.rstrip("/").split("/")[-1]
if name.endswith(".git"):
name = name[:-4]
name = "".join(c if c.isalnum() or c in "-_" else "_" for c in name)
return name or "repository"
async def _run_git(self, args: List[str], cwd: Optional[str] = None) -> str:
proc = await asyncio.create_subprocess_exec(
*args,
cwd=cwd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
if proc.returncode != 0:
error_msg = stderr.decode().strip()
user_msg = "Git command failed."
if "Could not resolve hostname" in error_msg:
user_msg = (
"Git command failed: could not resolve hostname. Check the URL or your network."
)
elif "Permission denied" in error_msg or "publickey" in error_msg:
user_msg = (
"Git command failed: authentication error. Check your SSH keys or credentials."
)
raise RuntimeError(
f"{user_msg} Command: git {' '.join(args[1:])}. Details: {error_msg}"
)
return stdout.decode().strip()
async def _has_commit(self, repo_dir: str, commit: str) -> bool:
try:
await self._run_git(["git", "-C", repo_dir, "rev-parse", "--verify", commit])
return True
except RuntimeError:
return False
@staticmethod
def _is_github_url(url: str) -> bool:
"""Return True for github.com URLs (supports ZIP archive API)."""
return is_github_url(url)
async def _github_zip_download(
self,
repo_url: str,
branch: Optional[str],
target_dir: str,
) -> Tuple[Path, str]:
"""Download a GitHub repo as a ZIP archive and extract it.
Uses the GitHub archive API (single HTTPS GET, no git history).
Returns:
(content_dir, repo_name) — content_dir is the extracted repo root.
"""
repo_name = self._get_repo_name(repo_url)
# Build archive URL from owner/repo path components.
parsed = urlparse(repo_url)
path_parts = [p for p in parsed.path.split("/") if p]
owner = path_parts[0]
repo_raw = path_parts[1]
# Strip .git suffix for the archive URL (git clone keeps it, ZIP API does not).
repo_slug = repo_raw[:-4] if repo_raw.endswith(".git") else repo_raw
if branch:
zip_url = f"https://github.com/{owner}/{repo_slug}/archive/{branch}.zip"
else:
zip_url = f"https://github.com/{owner}/{repo_slug}/archive/HEAD.zip"
logger.info(f"Downloading GitHub ZIP: {zip_url}")
zip_path = os.path.join(target_dir, "_archive.zip")
extract_dir = os.path.join(target_dir, "_extracted")
os.makedirs(extract_dir, exist_ok=True)
# Download (blocking HTTP; run in thread pool to avoid stalling event loop).
def _download() -> None:
headers = {"User-Agent": "OpenViking"}
github_token = os.environ.get("GITHUB_TOKEN")
if github_token:
headers["Authorization"] = f"token {github_token}"
req = urllib.request.Request(zip_url, headers=headers)
with urllib.request.urlopen(req, timeout=1800) as resp, open(zip_path, "wb") as f:
shutil.copyfileobj(resp, f)
try:
await asyncio.to_thread(_download)
except Exception as exc:
raise RuntimeError(f"Failed to download GitHub ZIP {zip_url}: {exc}")
# Safe extraction with Zip Slip validation (mirrors _extract_zip logic).
target = Path(extract_dir).resolve()
with zipfile.ZipFile(zip_path, "r") as zf:
for info in zf.infolist():
mode = info.external_attr >> 16
if info.is_dir() or stat.S_ISDIR(mode):
continue
if stat.S_ISLNK(mode):
logger.warning(f"Skipping symlink entry in GitHub ZIP: {info.filename}")
continue
raw = info.filename.replace("\\", "/")
raw_parts = [p for p in raw.split("/") if p]
if ".." in raw_parts:
raise ValueError(f"Zip Slip detected in GitHub archive: {info.filename!r}")
if PurePosixPath(raw).is_absolute():
raise ValueError(f"Zip Slip detected in GitHub archive: {info.filename!r}")
extracted = Path(zf.extract(info, extract_dir)).resolve()
if not extracted.is_relative_to(target):
extracted.unlink(missing_ok=True)
raise ValueError(f"Zip Slip detected in GitHub archive: {info.filename!r}")
# Remove downloaded archive to free disk space.
try:
os.unlink(zip_path)
except OSError:
pass
# GitHub ZIPs have a single top-level directory: {repo}-{branch}/ or {repo}-{sha}/.
# Return that directory as the content root so callers see bare repo files.
top_level = [d for d in Path(extract_dir).iterdir() if d.is_dir()]
content_dir = top_level[0] if len(top_level) == 1 else Path(extract_dir)
logger.info(f"GitHub ZIP extracted to {content_dir} ({repo_name})")
return content_dir, repo_name
async def _git_clone(
self,
url: str,
target_dir: str,
branch: Optional[str] = None,
commit: Optional[str] = None,
) -> str:
"""Clone a git repository into target_dir; return the repo name.
Uses --depth 1 for speed. If a specific commit is requested, it is
fetched and checked out after the shallow clone.
Returns:
Repository name derived from the URL (e.g. "OpenViking").
"""
name = self._get_repo_name(url)
logger.info(f"Cloning {url} to {target_dir}...")
clone_args = [
"git",
"clone",
"--depth",
"1",
"--no-recurse-submodules",
]
if branch and not commit:
clone_args.extend(["--branch", branch])
clone_args.extend([url, target_dir])
await self._run_git(clone_args)
if commit:
try:
await self._run_git(["git", "-C", target_dir, "fetch", "origin", commit])
except RuntimeError:
try:
await self._run_git(
["git", "-C", target_dir, "fetch", "--all", "--tags", "--prune"]
)
except RuntimeError:
pass
ok = await self._has_commit(target_dir, commit)
if not ok:
try:
await self._run_git(
["git", "-C", target_dir, "fetch", "--unshallow", "origin"]
)
except RuntimeError:
pass
ok = await self._has_commit(target_dir, commit)
if not ok:
await self._run_git(
[
"git",
"-C",
target_dir,
"fetch",
"origin",
"+refs/heads/*:refs/remotes/origin/*",
]
)
ok = await self._has_commit(target_dir, commit)
if not ok:
raise RuntimeError(f"Failed to fetch commit {commit} from {url}")
await self._run_git(["git", "-C", target_dir, "checkout", commit])
return name
async def _extract_zip(self, zip_path: str, target_dir: str) -> str:
"""Extract a local zip file into target_dir; return the archive stem as the repo name."""
if zip_path.startswith(("http://", "https://")):
# TODO: implement download logic or rely on caller?
# For now, assume it's implemented if needed, but raise error as strictly we only support git URL for now as per plan
raise NotImplementedError(
"Zip URL download not yet implemented in CodeRepositoryParser"
)
path = Path(zip_path)
name = path.stem
with zipfile.ZipFile(zip_path, "r") as zip_ref:
target = Path(target_dir).resolve()
for info in zip_ref.infolist():
mode = info.external_attr >> 16
# Skip directory entries (check both name convention and external attrs)
if info.is_dir() or stat.S_ISDIR(mode):
continue
# Skip symlink entries to prevent symlink-based escapes
if stat.S_ISLNK(mode):
logger.warning(f"Skipping symlink entry in zip: {info.filename}")
continue
# Reject entries with suspicious raw path components before extraction
raw = info.filename.replace("\\", "/")
raw_parts = [p for p in raw.split("/") if p]
if ".." in raw_parts:
raise ValueError(f"Zip Slip detected: entry {info.filename!r} contains '..'")
if PurePosixPath(raw).is_absolute() or (len(raw) >= 2 and raw[1] == ":"):
raise ValueError(
f"Zip Slip detected: entry {info.filename!r} is an absolute path"
)
# Normalize the member name the same way zipfile does
# (strip drive/UNC, remove empty/"."/ ".." components) then verify
arcname = info.filename.replace("/", os.sep)
if os.path.altsep:
arcname = arcname.replace(os.path.altsep, os.sep)
arcname = os.path.splitdrive(arcname)[1]
arcname = os.sep.join(p for p in arcname.split(os.sep) if p not in ("", ".", ".."))
if not arcname:
continue # entry normalizes to empty path, skip
member_path = (Path(target_dir) / arcname).resolve()
if not member_path.is_relative_to(target):
raise ValueError(
f"Zip Slip detected: entry {info.filename!r} escapes target directory"
)
# Extract single member and verify the actual path on disk
extracted = Path(zip_ref.extract(info, target_dir)).resolve()
if not extracted.is_relative_to(target):
# Best-effort cleanup of the escaped file
try:
extracted.unlink(missing_ok=True)
except OSError as cleanup_err:
logger.warning(
f"Failed to clean up escaped file {extracted}: {cleanup_err}"
)
raise ValueError(
f"Zip Slip detected: entry {info.filename!r} escapes target directory"
)
return name
async def _upload_directory(
self,
local_dir: Path,
*,
output_store: Any,
artifact_ref: Any,
ignore_dirs: Optional[Union[Set[str], List[str], str]] = None,
include: Optional[str] = None,
exclude: Optional[str] = None,
) -> Tuple[int, List[str]]:
"""Recursively upload the repository into the artifact store under repository/."""
return await upload_directory(
local_dir,
"repository",
store=output_store,
artifact_ref=artifact_ref,
ignore_dirs=ignore_dirs,
include=include,
exclude=exclude,
)