Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat: allow private gitlab domain for code repo (#285)
MaojiaShengc943fb0ea954
diff --git a/examples/ov.conf.example b/examples/ov.conf.exampleindex 802ce8491..0392ddb36 100644--- a/examples/ov.conf.example+++ b/examples/ov.conf.example@@ -94,8 +94,12 @@"max_line_length": 1000,"max_token_limit": 50000,"truncation_strategy": "head",- "warn_on_truncation": true- },+ "warn_on_truncation": true,+ "github_raw_domain": "raw.githubusercontent.com",+ "code_hosting_domains": ["github.com", "gitlab.com"],+ "github_domains": ["github.com", "www.github.com"],+ "gitlab_domains": ["gitlab.com", "www.gitlab.com"]+ }"image": {"enable_ocr": false,"enable_vlm": true,@@ -126,8 +130,11 @@"extract_text_only": false,"preserve_structure": true,"clean_html": true,- "extract_metadata": true- },+ "extract_metadata": true,+ "code_hosting_domains": ["github.com", "gitlab.com"],+ "github_domains": ["github.com", "www.github.com"],+ "gitlab_domains": ["gitlab.com", "www.gitlab.com"]+ }"text": {"detect_language": true,"split_by_paragraphs": true,diff --git a/openviking/parse/parsers/code/code.py b/openviking/parse/parsers/code/code.pyindex 3693384b6..de7580ec0 100644--- a/openviking/parse/parsers/code/code.py+++ b/openviking/parse/parsers/code/code.py@@ -38,6 +38,8 @@ from openviking.parse.parsers.constants import (IGNORE_EXTENSIONS,)from openviking.parse.parsers.upload_utils import upload_directory+from openviking.utils import is_github_url, parse_code_hosting_url+from openviking_cli.utils.config import get_openviking_configfrom openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)@@ -132,8 +134,9 @@ class CodeRepositoryParser(BaseParser):# 3. Create VikingFS temp URIviking_fs = self._get_viking_fs()temp_viking_uri = self._create_temp_uri()- # The structure in temp should be: viking://temp/{uuid}/{repo_name}/...- target_root_uri = f"{temp_viking_uri}/{repo_name}"+ # The structure in temp should be: viking://temp/{uuid}/repository/...+ # Use simple name 'repository' for temp, TreeBuilder will rename it to org/repo later+ target_root_uri = f"{temp_viking_uri}/repository"logger.info(f"Uploading to VikingFS: {target_root_uri}")@@ -243,18 +246,36 @@ class CodeRepositoryParser(BaseParser):git_index = next((i for i, p in enumerate(path_parts) if p.endswith(".git")), None)if git_index is not None:base_parts = path_parts[: git_index + 1]- elif parsed.netloc in ["github.com", "gitlab.com"] and len(path_parts) >= 2:++ config = get_openviking_config()+ if (+ parsed.netloc in config.code.github_domains + config.code.gitlab_domains+ and len(path_parts) >= 2+ ):base_parts = path_parts[:2]base_path = "/" + "/".join(base_parts)return parsed._replace(path=base_path, query="", fragment="").geturl()return urldef _get_repo_name(self, url: str) -> str:+ """Get repository name with organization for GitHub/GitLab URLs.++ For https://github.com/volcengine/OpenViking, returns "volcengine/OpenViking"+ For other URLs, falls back to just the repo name.+ """+ # First try to parse as code hosting URL+ parsed_org_repo = parse_code_hosting_url(url)+ if parsed_org_repo:+ return parsed_org_repo++ # Fallback for other URLsname_source = urlif url.startswith(("http://", "https://", "git://", "ssh://")):name_source = urlparse(url).path.rstrip("/")elif ":" in url and not url.startswith("file://"):name_source = url.split(":", 1)[1]++ # Original logic for non-GitHub/GitLab URLsname = name_source.rstrip("/").split("/")[-1]if name.endswith(".git"):name = name[:-4]@@ -284,7 +305,7 @@ class CodeRepositoryParser(BaseParser):@staticmethoddef _is_github_url(url: str) -> bool:"""Return True for github.com URLs (supports ZIP archive API)."""- return urlparse(url).netloc in ("github.com", "www.github.com")+ return is_github_url(url)async def _github_zip_download(self,diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.pyindex 3b9739fbb..48ae4fa7b 100644--- a/openviking/parse/parsers/html.py+++ b/openviking/parse/parsers/html.py@@ -28,6 +28,7 @@ from openviking.parse.base import (lazy_import,)from openviking.parse.parsers.base_parser import BaseParser+from openviking_cli.utils.config import get_openviking_configclass URLType(Enum):@@ -141,13 +142,27 @@ class URLTypeDetector:"""import re- # Repository URL patterns (from README)- repo_patterns = [- r"^https?://github\.com/[^/]+/[^/]+/?$",- r"^https?://gitlab\.com/[^/]+/[^/]+/?$",- r"^.*\.git$",- r"^git@",- ]+ config = get_openviking_config()+ github_domains = list(set(config.html.github_domains + config.code.github_domains))+ gitlab_domains = list(set(config.html.gitlab_domains + config.code.gitlab_domains))+ # Build repository URL patterns from config+ repo_patterns = []++ # Add patterns for GitHub domains+ for domain in github_domains:+ repo_patterns.append(rf"^https?://{re.escape(domain)}/[^/]+/[^/]+/?$")++ # Add patterns for GitLab domains+ for domain in gitlab_domains:+ repo_patterns.append(rf"^https?://{re.escape(domain)}/[^/]+/[^/]+/?$")++ # Add other patterns+ repo_patterns.extend(+ [+ r"^.*\.git$",+ r"^git@",+ ]+ )# Check for URL patternsfor pattern in repo_patterns:@@ -478,15 +493,19 @@ class HTMLParser(BaseParser):def _convert_to_raw_url(self, url: str) -> str:"""Convert GitHub/GitLab blob URL to raw URL."""parsed = urlparse(url)+ config = get_openviking_config()+ github_domains = config.html.github_domains+ gitlab_domains = config.html.gitlab_domains+ github_raw_domain = config.code.github_raw_domain- if parsed.netloc == "github.com":+ if parsed.netloc in github_domains:path_parts = parsed.path.strip("/").split("/")if len(path_parts) >= 4 and path_parts[2] == "blob":# Remove 'blob'new_path = "/".join(path_parts[:2] + path_parts[3:])- return f"https://raw.githubusercontent.com/{new_path}"+ return f"https://{github_raw_domain}/{new_path}"- if parsed.netloc == "gitlab.com" and "/blob/" in parsed.path:+ if parsed.netloc in gitlab_domains and "/blob/" in parsed.path:return url.replace("/blob/", "/raw/")return urldiff --git a/openviking/parse/tree_builder.py b/openviking/parse/tree_builder.pyindex 9a767fce7..5efbad3a4 100644--- a/openviking/parse/tree_builder.py+++ b/openviking/parse/tree_builder.py@@ -29,6 +29,7 @@ from openviking.parse.parsers.media.utils import get_media_base_uri, get_media_tfrom openviking.server.identity import RequestContextfrom openviking.storage.queuefs import SemanticMsg, get_queue_managerfrom openviking.storage.viking_fs import get_viking_fs+from openviking.utils import parse_code_hosting_urlfrom openviking_cli.utils.uri import VikingURIif TYPE_CHECKING:@@ -129,13 +130,40 @@ class TreeBuilder:doc_name = VikingURI.sanitize_segment(doc_dirs[0]["name"])temp_doc_uri = f"{temp_uri}/{doc_name}"- # 2. Determine base_uri+ # 2. Determine base_uri and final document name with org/repo for GitHub/GitLabif base_uri is None:base_uri = self._get_base_uri(scope, source_path, source_format)- # 3. Build final URI, auto-renaming on conflict (e.g. doc_1, doc_2, ...)- candidate_uri = VikingURI(base_uri).join(doc_name).uri+ # Check if source_path is a GitHub/GitLab URL and extract org/repo+ final_doc_name = doc_name+ if source_path and source_format == "repository":+ parsed_org_repo = parse_code_hosting_url(source_path)+ if parsed_org_repo:+ final_doc_name = parsed_org_repo++ # 3. Check if base_uri exists - if it does, use it as parent directory+ try:+ await viking_fs.stat(base_uri)+ base_exists = True+ except Exception:+ base_exists = False++ if base_exists:+ if "/" in final_doc_name:+ repo_name_only = final_doc_name.split("/")[-1]+ else:+ repo_name_only = final_doc_name+ candidate_uri = VikingURI(base_uri).join(repo_name_only).uri+ else:+ if "/" in final_doc_name:+ parts = final_doc_name.split("/")+ sanitized_parts = [VikingURI.sanitize_segment(p) for p in parts if p]+ base_viking_uri = VikingURI(base_uri)+ candidate_uri = VikingURI.build(base_viking_uri.scope, *sanitized_parts)+ else:+ candidate_uri = VikingURI(base_uri).join(doc_name).urifinal_uri = await self._resolve_unique_uri(candidate_uri, ctx=ctx)+if final_uri != candidate_uri:logger.info(f"[TreeBuilder] Resolved name conflict: {candidate_uri} -> {final_uri}")else:diff --git a/openviking/utils/__init__.py b/openviking/utils/__init__.pyindex a86f76ff9..be6def670 100644--- a/openviking/utils/__init__.py+++ b/openviking/utils/__init__.py@@ -2,6 +2,12 @@# SPDX-License-Identifier: Apache-2.0"""Utility functions and helpers."""+from openviking.utils.code_hosting_utils import (+ is_code_hosting_url,+ is_github_url,+ is_gitlab_url,+ parse_code_hosting_url,+)from openviking.utils.time_utils import get_current_timestampfrom openviking_cli.utils.async_utils import run_asyncfrom openviking_cli.utils.llm import StructuredLLM, parse_json_from_response, parse_json_to_model@@ -17,4 +23,8 @@ __all__ = ["parse_json_from_response","parse_json_to_model","run_async",+ "parse_code_hosting_url",+ "is_github_url",+ "is_gitlab_url",+ "is_code_hosting_url",]diff --git a/openviking/utils/code_hosting_utils.py b/openviking/utils/code_hosting_utils.pynew file mode 100644index 000000000..eab27a2a2--- /dev/null+++ b/openviking/utils/code_hosting_utils.py@@ -0,0 +1,95 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""+Utilities for code hosting platform URL parsing.++This module provides shared functionality for parsing URLs from code hosting+platforms like GitHub and GitLab.+"""++from typing import Optional+from urllib.parse import urlparse++from openviking_cli.utils.config import get_openviking_config+++def parse_code_hosting_url(url: str) -> Optional[str]:+ """Parse code hosting platform URL to get org/repo path.++ Args:+ url: Code hosting URL like https://github.com/volcengine/OpenViking++ Returns:+ org/repo path like "volcengine/OpenViking" or None if not a valid+ code hosting URL+ """+ if not url.startswith(("http://", "https://", "git://", "ssh://")):+ return None++ parsed = urlparse(url)+ path_parts = [p for p in parsed.path.split("/") if p]++ config = get_openviking_config()++ # For GitHub/GitLab URLs with org/repo structure+ if (+ parsed.netloc in config.code.github_domains + config.code.gitlab_domains+ and len(path_parts) >= 2+ ):+ # Take first two parts: org/repo+ org = path_parts[0]+ repo = path_parts[1]+ if repo.endswith(".git"):+ repo = repo[:-4]+ # Sanitize both parts+ org = "".join(c if c.isalnum() or c in "-_" else "_" for c in org)+ repo = "".join(c if c.isalnum() or c in "-_" else "_" for c in repo)+ return f"{org}/{repo}"++ return None+++def is_github_url(url: str) -> bool:+ """Check if a URL is a GitHub URL.++ Args:+ url: URL to check++ Returns:+ True if the URL is a GitHub URL+ """+ config = get_openviking_config()+ return urlparse(url).netloc in config.code.github_domains+++def is_gitlab_url(url: str) -> bool:+ """Check if a URL is a GitLab URL.++ Args:+ url: URL to check++ Returns:+ True if the URL is a GitLab URL+ """+ config = get_openviking_config()+ return urlparse(url).netloc in config.code.gitlab_domains+++def is_code_hosting_url(url: str) -> bool:+ """Check if a URL is a code hosting platform URL.++ Args:+ url: URL to check++ Returns:+ True if the URL is a code hosting platform URL+ """+ config = get_openviking_config()+ all_domains = list(+ set(+ config.code.github_domains+ + config.code.gitlab_domains+ + config.code.code_hosting_domains+ )+ )+ return urlparse(url).netloc in all_domainsdiff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.pyindex 106a33c85..c36a4ced0 100644--- a/openviking/utils/resource_processor.py+++ b/openviking/utils/resource_processor.py@@ -127,6 +127,9 @@ class ResourceProcessor:result["status"] = "error"result["errors"].append(f"Parse error: {e}")logger.error(f"[ResourceProcessor] Parse error: {e}")+ import traceback++ traceback.print_exc()return result# parse_result contains:diff --git a/openviking_cli/utils/config/parser_config.py b/openviking_cli/utils/config/parser_config.pyindex 079daf533..a5bcd10b6 100644--- a/openviking_cli/utils/config/parser_config.py+++ b/openviking_cli/utils/config/parser_config.py@@ -135,7 +135,7 @@ class PDFConfig(ParserConfig):mineru_endpoint: MinerU API endpoint URLmineru_api_key: MinerU API authentication keymineru_timeout: MinerU request timeout in seconds- mineruparams: Additional MinerU API parameters+ mineru_params: Additional MinerU API parameters"""strategy: str = "auto" # "local" | "mineru" | "auto"@@ -171,7 +171,33 @@ class PDFConfig(ParserConfig):@dataclass-class CodeConfig(ParserConfig):+class CodeHostingConfig(ParserConfig):+ """+ Base configuration for code hosting platform domains.++ Attributes:+ code_hosting_domains: List of code hosting platform domains (github.com, gitlab.com, etc.)+ github_domains: List of GitHub domains (github.com, www.github.com)+ gitlab_domains: List of GitLab domains (gitlab.com, www.gitlab.com)+ """++ # Code hosting platform configuration+ code_hosting_domains: list = None+ github_domains: list = None+ gitlab_domains: list = None++ def __post_init__(self):+ """Initialize default values for mutable fields."""+ if self.code_hosting_domains is None:+ self.code_hosting_domains = ["github.com", "gitlab.com"]+ if self.github_domains is None:+ self.github_domains = ["github.com", "www.github.com"]+ if self.gitlab_domains is None:+ self.gitlab_domains = ["gitlab.com", "www.gitlab.com"]+++@dataclass+class CodeConfig(CodeHostingConfig):"""Configuration for code parsing.@@ -186,6 +212,7 @@ class CodeConfig(ParserConfig):max_token_limit: Maximum tokens to process per filetruncation_strategy: "head", "tail", or "balanced"warn_on_truncation: Whether to warn when truncation occurs+ github_raw_domain: Domain for GitHub raw content (raw.githubusercontent.com)"""enable_ast: bool = True@@ -198,6 +225,7 @@ class CodeConfig(ParserConfig):max_token_limit: int = 50000 # Maximum tokens to process per filetruncation_strategy: str = "head" # "head", "tail", or "balanced"warn_on_truncation: bool = True+ github_raw_domain: str = "raw.githubusercontent.com"def validate(self) -> None:"""@@ -359,7 +387,7 @@ class MarkdownConfig(ParserConfig):@dataclass-class HTMLConfig(ParserConfig):+class HTMLConfig(CodeHostingConfig):"""Configuration for HTML parsing.