Skip to content

Create, Evaluate, and Connect AI Skills

update downloader

LiamLYYYa0b10ad77abc
diff --git a/skillnet-ai/pyproject.toml b/skillnet-ai/pyproject.toml
index a80dcdf..ca48621 100644
--- a/skillnet-ai/pyproject.toml
+++ b/skillnet-ai/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "skillnet-ai"
-version = "0.0.13"
+version = "0.0.14"
description = "Official Python SDK for SkillNet: Create, Evaluate, and Connect AI Skills."
readme = "README.md"
authors = [{ name = "SkillNet Team", email = "[email protected]" }]
diff --git a/skillnet-ai/src/skillnet_ai/downloader.py b/skillnet-ai/src/skillnet_ai/downloader.py
index 77ea3f4..b545109 100644
--- a/skillnet-ai/src/skillnet_ai/downloader.py
+++ b/skillnet-ai/src/skillnet_ai/downloader.py
@@ -1,7 +1,7 @@
-import requests
import os
import logging
-from typing import Optional, List, Dict
+import requests
+from typing import Optional, List, Dict, Tuple
logger = logging.getLogger(__name__)
@@ -13,144 +13,219 @@ class SkillDownloader:
def __init__(self, api_token: Optional[str] = None):
"""
- Initialize the downloader.
-
+ Initializes the SkillDownloader.
+
Args:
- api_token: Optional GitHub Personal Access Token to avoid rate limits
- and access private repositories.
+ api_token (Optional[str]): GitHub Personal Access Token to increase
+ rate limits and access private repositories.
"""
self.api_token = api_token
self.session = requests.Session()
self.session.headers.update({
"Accept": "application/vnd.github.v3+json"
})
+
if self.api_token:
self.session.headers.update({"Authorization": f"token {self.api_token}"})
def download(self, folder_url: str, target_dir: str = ".") -> Optional[str]:
"""
- Download a specific GitHub subdirectory to the target directory.
-
+ Downloads a specific GitHub subdirectory or file to the target directory.
+
Args:
- folder_url: The full URL to the GitHub folder.
- target_dir: The local directory to install the skill into.
-
+ folder_url (str): The full URL to the GitHub folder or file
+ (e.g., tree or blob URL).
+ target_dir (str): The local directory where the skill will be installed.
+
Returns:
- The absolute path to the installed skill directory, or None if failed.
+ Optional[str]: The absolute path to the installed skill directory,
+ or None if the download fails.
"""
try:
- # 1. Parse the URL
+ # 1. Parse the GitHub URL
parsed_info = self._parse_github_url(folder_url)
if not parsed_info:
return None
owner, repo, ref, dir_path, folder_name = parsed_info
- logger.info(f"Parsing: {owner}/{repo} @ {ref} -> {dir_path}")
+ logger.info(f"Parsing repository details: {owner}/{repo} @ {ref} -> {dir_path}")
- # 2. Get file tree
+ # 2. Fetch the file tree for the specific directory
files_to_download = self._get_file_tree(owner, repo, ref, dir_path)
if not files_to_download:
- logger.warning("No matching files found or API error.")
+ logger.warning(f"No matching files found or API error for path: {dir_path}")
return None
- logger.info(f"Found {len(files_to_download)} files, starting download...")
+ logger.info(f"Identified {len(files_to_download)} file(s). Starting download...")
- # 3. Download files
+ # 3. Download the identified files
success_count = 0
failed_files = []
+
for file_info in files_to_download:
- if self._download_single_file(owner, repo, ref, dir_path, file_info, folder_name, target_dir):
+ is_success = self._download_single_file(
+ owner, repo, ref, dir_path, file_info, folder_name, target_dir
+ )
+ if is_success:
success_count += 1
else:
- failed_files.append(file_info['path'])
+ failed_files.append(file_info.get("path", "Unknown path"))
+ # 4. Handle results and determine final installation path
if success_count == 0:
- logger.error("❌ Failed to download any files. Please check your network settings and ensure connection to GitHub is working properly.")
+ logger.error("❌ Failed to download any files. Please verify network connectivity and GitHub API limits.")
return None
final_path = os.path.abspath(os.path.join(target_dir, folder_name))
if failed_files:
- logger.warning(f"⚠️ Successfully downloaded {success_count} files at {final_path}, but {len(failed_files)} failed.")
+ logger.warning(f"⚠️ Successfully downloaded {success_count} file(s) to {final_path}, but {len(failed_files)} failed.")
logger.warning("The following files could not be downloaded:")
for f in failed_files:
logger.warning(f" - {f}")
- logger.warning("Please check your network settings and ensure connection to GitHub is working properly.")
else:
logger.info(f"✅ Skill installed successfully at: {final_path}")
+
return final_path
except Exception as e:
- logger.error(f"Critical error during installation: {e}")
+ logger.error(f"Critical error during skill installation: {e}")
return None
- def _parse_github_url(self, url: str) -> Optional[tuple]:
+ def _parse_github_url(self, url: str) -> Optional[Tuple[str, str, str, str, str]]:
"""
- Parses the GitHub URL into components.
- Expected format: https://github.com/owner/repo/tree/ref/path/to/dir
+ Parses a standard GitHub tree or blob URL into its constituent components.
+
+ Args:
+ url (str): The GitHub URL to parse.
+
+ Returns:
+ Optional[Tuple]: A tuple containing (owner, repo, ref, dir_path, folder_name),
+ or None if the URL is invalid.
"""
parts = url.rstrip('/').split('/')
if len(parts) < 7:
- logger.error(f"Invalid GitHub URL format: {url}")
+ logger.error(f"Invalid GitHub URL format provided: {url}")
return None
- # parts[2] is usually 'github.com'
- owner, repo = parts[3], parts[4]
- ref = parts[6] # branch or commit hash
+ # Assume parts[2] is 'github.com'
+ owner = parts[3]
+ repo = parts[4]
+ ref = parts[6] # branch name or commit hash
dir_path = "/".join(parts[7:])
folder_name = parts[-1]
return owner, repo, ref, dir_path, folder_name
- def _get_file_tree(self, owner: str, repo: str, ref: str, dir_path: str) -> List[Dict]:
+ def _get_file_tree(self, owner: str, repo: str, ref: str, dir_path: str) -> List[Dict[str, str]]:
"""
- Fetches the recursive file tree from GitHub API and filters for the target directory.
+ Fetches the file list using the GitHub Contents API.
+ Recursively resolves subdirectories while bypassing repository-wide limits.
+
+ Args:
+ owner (str): Repository owner.
+ repo (str): Repository name.
+ ref (str): Branch or commit reference.
+ dir_path (str): The target directory or file path within the repository.
+
+ Returns:
+ List[Dict[str, str]]: A list of dictionaries containing file 'path' and 'download_url'.
"""
- api_url = f"https://api.github.com/repos/{owner}/{repo}/git/trees/{ref}?recursive=1"
+ files_to_download = []
+ api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/{dir_path}?ref={ref}"
- response = self.session.get(api_url)
- if response.status_code != 200:
- logger.error(f"GitHub API error: {response.status_code} - {response.text}")
- return []
+ try:
+ response = self.session.get(api_url)
+ if response.status_code != 200:
+ logger.error(f"GitHub API Error [{response.status_code}]: {response.text}")
+ return []
- tree = response.json().get('tree', [])
-
- # Filter files that are inside the target dir_path and are blobs (files, not folders)
- return [
- item for item in tree
- if item['path'].startswith(dir_path) and item['type'] == 'blob'
- ]
+ contents = response.json()
+
+ # Handle single file scenario (e.g., blob URL)
+ if isinstance(contents, dict) and contents.get("type") == "file":
+ return [{
+ "path": contents.get("path", ""),
+ "download_url": contents.get("download_url", "")
+ }]
- def _download_single_file(self, owner: str, repo: str, ref: str, dir_path: str, file_info: Dict, folder_name: str, target_dir: str) -> bool:
+ # Handle directory scenario (e.g., tree URL)
+ if isinstance(contents, list):
+ for item in contents:
+ if item.get("type") == "file":
+ files_to_download.append({
+ "path": item.get("path", ""),
+ "download_url": item.get("download_url", "")
+ })
+ elif item.get("type") == "dir":
+ # Recursively fetch contents of subdirectories
+ sub_path = item.get("path", "")
+ sub_files = self._get_file_tree(owner, repo, ref, sub_path)
+ files_to_download.extend(sub_files)
+
+ return files_to_download
+
+ except Exception as e:
+ logger.error(f"Failed to retrieve file tree for {dir_path}: {e}")
+ return []
+
+ def _download_single_file(
+ self,
+ owner: str,
+ repo: str,
+ ref: str,
+ dir_path: str,
+ file_info: Dict[str, str],
+ folder_name: str,
+ target_dir: str
+ ) -> bool:
"""
- Downloads a single file from raw.githubusercontent.com.
+ Downloads a single file from GitHub and saves it to the local target directory.
+
+ Args:
+ owner (str): Repository owner.
+ repo (str): Repository name.
+ ref (str): Branch or commit reference.
+ dir_path (str): The base directory path requested by the user.
+ file_info (Dict[str, str]): File metadata containing 'path' and 'download_url'.
+ folder_name (str): The root folder name for local installation.
+ target_dir (str): The local target directory.
+
+ Returns:
+ bool: True if the file was downloaded and saved successfully, False otherwise.
"""
- # Construct raw URL
- # Note: Handling private repos might require using the API content endpoint instead of raw.githubusercontent
- raw_url = f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{file_info['path']}"
+ raw_url = file_info.get("download_url")
+ file_path = file_info.get("path", "")
+
+ # Fallback for raw content URL if download_url is unexpectedly missing
+ if not raw_url:
+ raw_url = f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{file_path}"
- # Calculate local path
- relative_path = file_info['path'].replace(f"{dir_path}/", "")
- # Remove leading slash if present to avoid path join issues
- relative_path = relative_path.lstrip('/')
+ # Calculate the relative path for the local file system
+ if file_path == dir_path:
+ # Exact file match (blob download)
+ relative_path = os.path.basename(file_path)
+ else:
+ # Directory match: strip the base requested directory from the path
+ relative_path = file_path.replace(f"{dir_path}/", "", 1)
+ relative_path = relative_path.lstrip('/')
local_file_path = os.path.join(target_dir, folder_name, relative_path)
- # Create parent directories
+ # Ensure parent directories exist
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
try:
- # If it's a private repo, raw.githubusercontent might need token in header
- # The session already has the token if provided.
- file_resp = self.session.get(raw_url)
+ response = self.session.get(raw_url)
- if file_resp.status_code == 200:
+ if response.status_code == 200:
with open(local_file_path, "wb") as f:
- f.write(file_resp.content)
+ f.write(response.content)
return True
else:
- logger.warning(f"Failed to download {raw_url}: {file_resp.status_code}")
+ logger.warning(f"Failed to download {raw_url} - HTTP Status: {response.status_code}")
return False
+
except Exception as e:
- logger.error(f"Exception downloading {raw_url}: {e}")
+ logger.error(f"Network or file system exception while processing {raw_url}: {e}")
return False
\ No newline at end of file