Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: fix handle github url right (#180)

wangxiaolei68c66880fa99
diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.py
index 7e867a3e2..28e47885f 100644
--- a/openviking/parse/parsers/html.py
+++ b/openviking/parse/parsers/html.py
@@ -153,13 +153,6 @@ class URLTypeDetector:
if re.match(pattern, url):
return True
- # Check if it's a GitHub/GitLab URL
- parsed = urlparse(url)
- if parsed.netloc in ["github.com", "gitlab.com"]:
- path_parts = parsed.path.strip("/").split("/")
- if len(path_parts) >= 2:
- return True
-
return False
@@ -359,14 +352,17 @@ class HTMLParser(BaseParser):
# Get appropriate parser
if file_type == "pdf":
from openviking.parse.parsers.pdf import PDFParser
+
parser = PDFParser()
result = await parser.parse(temp_path)
elif file_type == "markdown":
from openviking.parse.parsers.markdown import MarkdownParser
+
parser = MarkdownParser()
result = await parser.parse(temp_path)
elif file_type == "text":
from openviking.parse.parsers.text import TextParser
+
parser = TextParser()
result = await parser.parse(temp_path)
elif file_type == "html":
@@ -478,6 +474,22 @@ class HTMLParser(BaseParser):
response.raise_for_status()
return response.text
+ def _convert_to_raw_url(self, url: str) -> str:
+ """Convert GitHub/GitLab blob URL to raw URL."""
+ parsed = urlparse(url)
+
+ if parsed.netloc == "github.com":
+ path_parts = parsed.path.strip("/").split("/")
+ if len(path_parts) >= 4 and path_parts[2] == "blob":
+ # Remove 'blob'
+ new_path = "/".join(path_parts[:2] + path_parts[3:])
+ return f"https://raw.githubusercontent.com/{new_path}"
+
+ if parsed.netloc == "gitlab.com" and "/blob/" in parsed.path:
+ return url.replace("/blob/", "/raw/")
+
+ return url
+
async def _download_file(self, url: str) -> str:
"""
Download file from URL to temporary location.
@@ -493,6 +505,8 @@ class HTMLParser(BaseParser):
"""
httpx = lazy_import("httpx")
+ url = self._convert_to_raw_url(url)
+
# Determine file extension from URL
parsed = urlparse(url)
ext = Path(parsed.path).suffix or ".tmp"
diff --git a/tests/parse/test_html_parser_utils.py b/tests/parse/test_html_parser_utils.py
new file mode 100644
index 000000000..ca2373e7b
--- /dev/null
+++ b/tests/parse/test_html_parser_utils.py
@@ -0,0 +1,53 @@
+import pytest
+from openviking.parse.parsers.html import HTMLParser
+
+
+class TestHTMLParserRawUrlConversion:
+ """Test suite for HTMLParser._convert_to_raw_url method."""
+
+ def setup_method(self):
+ self.parser = HTMLParser()
+
+ def test_github_blob_conversion(self):
+ blob_url = "https://github.com/volcengine/OpenViking/blob/main/docs/design.md"
+ expected = "https://raw.githubusercontent.com/volcengine/OpenViking/main/docs/design.md"
+ assert self.parser._convert_to_raw_url(blob_url) == expected
+
+ blob_deep = "https://github.com/user/repo/blob/feature/branch/src/components/Button.tsx"
+ expected_deep = (
+ "https://raw.githubusercontent.com/user/repo/feature/branch/src/components/Button.tsx"
+ )
+ assert self.parser._convert_to_raw_url(blob_deep) == expected_deep
+
+ def test_github_non_blob_urls(self):
+ repo_root = "https://github.com/volcengine/OpenViking"
+ assert self.parser._convert_to_raw_url(repo_root) == repo_root
+
+ issue_url = "https://github.com/volcengine/OpenViking/issues/1"
+ assert self.parser._convert_to_raw_url(issue_url) == issue_url
+
+ raw_url = "https://raw.githubusercontent.com/volcengine/OpenViking/main/README.md"
+ assert self.parser._convert_to_raw_url(raw_url) == raw_url
+
+ def test_gitlab_blob_conversion(self):
+ blob_url = "https://gitlab.com/gitlab-org/gitlab/-/blob/master/README.md"
+ expected = "https://gitlab.com/gitlab-org/gitlab/-/raw/master/README.md"
+ assert self.parser._convert_to_raw_url(blob_url) == expected
+
+ blob_deep = "https://gitlab.com/group/project/-/blob/dev/src/main.rs"
+ expected_deep = "https://gitlab.com/group/project/-/raw/dev/src/main.rs"
+ assert self.parser._convert_to_raw_url(blob_deep) == expected_deep
+
+ def test_gitlab_non_blob_urls(self):
+ root = "https://gitlab.com/gitlab-org/gitlab"
+ assert self.parser._convert_to_raw_url(root) == root
+
+ issue = "https://gitlab.com/gitlab-org/gitlab/-/issues/123"
+ assert self.parser._convert_to_raw_url(issue) == issue
+
+ def test_other_domains(self):
+ url = "https://example.com/blob/main/file.txt"
+ assert self.parser._convert_to_raw_url(url) == url
+
+ bitbucket = "https://bitbucket.org/user/repo/src/master/README.md"
+ assert self.parser._convert_to_raw_url(bitbucket) == bitbucket