The definitive list of the best of (consumer facing) open source.
Removed the scripts directory
Lucas7e79fc06cf27
diff --git a/scripts/utils/contributing_autofill.py b/scripts/utils/contributing_autofill.pydeleted file mode 100644index 6096e5d..0000000--- a/scripts/utils/contributing_autofill.py+++ /dev/null@@ -1,433 +0,0 @@-import argparse-import base64-import json-import os-import re-import sys-from dataclasses import dataclass-from datetime import datetime-from pathlib import Path-from typing import Dict, List, Optional, Sequence, Set, Tuple-import requests--ROOT = Path(__file__).resolve().parents[2]-TAGS_FILE = ROOT / "source" / "data" / "static" / "tags.json"-PLATFORMS_FILE = ROOT / "source" / "data" / "static" / "platforms.json"-CATEGORIES_FILE = ROOT / "source" / "data" / "static" / "categories.json"-APPLICATIONS_FILE = ROOT / "source" / "data" / "dynamic" / "applications.json"-USER_AGENT = "definitive-opensource-contributing-autofill"-GITHUB_API_VERSION = "2022-11-28"--REPO_PATTERN = re.compile(- r"(?:github\.com[:/])?(?P<owner>[\w\-.]+)/(?P<repo>[\w\-.]+?)(?:\.git)?(?:[#?].*)?$",- re.IGNORECASE,-)--PLATFORM_KEYWORDS: Dict[str, str] = {}--TAG_KEYWORDS: Dict[str, str] = {}--CATEGORY_KEYWORDS: Dict[str, str] = {}---@dataclass-class ReferenceData:- tag_ids: Set[str]- platform_ids: Set[str]- category_ids: Set[str]- tag_labels: Dict[str, str]- platform_labels: Dict[str, str]- category_labels: Dict[str, str]---class DuplicateRepositoryError(RuntimeError):- """Raised when attempting to append an application that already exists."""---def load_reference_data() -> ReferenceData:- with open(TAGS_FILE, "r", encoding="utf-8") as fh:- tags_data = json.load(fh)- tag_labels: Dict[str, str] = {}- for entry in tags_data.get("attributes", []):- label = entry.get("description") or entry.get("name") or entry["id"]- emoji = entry.get("emoji")- if emoji:- label = f"{emoji} {label}"- tag_labels[entry["id"]] = label- for entry in tags_data.get("properties", []):- label = entry.get("name") or entry.get("description") or entry["id"]- tag_labels[entry["id"]] = label- tag_ids = set(tag_labels.keys())-- with open(PLATFORMS_FILE, "r", encoding="utf-8") as fh:- platforms_data = json.load(fh)- platform_labels = {- entry["id"]: entry.get("name") or entry["id"] for entry in platforms_data.get("platforms", [])- }- platform_ids = set(platform_labels.keys())-- with open(CATEGORIES_FILE, "r", encoding="utf-8") as fh:- categories_data = json.load(fh)- category_labels: Dict[str, str] = {- entry["id"]: entry.get("name") or entry["id"] for entry in categories_data.get("categories", [])- }- for entry in categories_data.get("subcategories", []):- parent = entry.get("parent")- parent_label = category_labels.get(parent, parent) if parent else None- name = entry.get("name") or entry["id"]- label = f"{name} ({parent_label})" if parent_label else name- category_labels[entry["id"]] = label- category_ids = set(category_labels.keys())-- return ReferenceData(- tag_ids=tag_ids,- platform_ids=platform_ids,- category_ids=category_ids,- tag_labels=tag_labels,- platform_labels=platform_labels,- category_labels=category_labels,- )---def render_options(options: Dict[str, str], indent: str = " ") -> str:- lines = []- for key, label in sorted(options.items()):- descriptor = f"{key}: {label}" if label and label != key else key- lines.append(f"{indent}- {descriptor}")- return "\n".join(lines)---def available_text(label: str, options: Dict[str, str]) -> str:- if not options:- return ""- return f"Available {label} ids:\n{render_options(options)}"---def load_applications_data(path: Path) -> Dict:- with open(path, "r", encoding="utf-8") as fh:- return json.load(fh)---def persist_applications_data(path: Path, data: Dict) -> None:- with open(path, "w", encoding="utf-8") as fh:- json.dump(data, fh, indent=4)- fh.write("\n")---def append_application(entry: Dict, path: Path) -> None:- data = load_applications_data(path)- applications = data.setdefault("applications", [])- new_url = (entry.get("repo_url") or "").rstrip("/")- for existing in applications:- if (existing.get("repo_url") or "").rstrip("/") == new_url:- try:- display_path = path.relative_to(ROOT)- except ValueError:- display_path = path- raise DuplicateRepositoryError(- f"Repository {entry['repo_url']} already exists in {display_path}."- )- applications.append(entry)- persist_applications_data(path, data)---def parse_repo_identifier(value: str) -> Tuple[str, str]:- value = value.strip()- match = REPO_PATTERN.search(value)- if match:- return match.group("owner"), match.group("repo")- if "/" in value:- owner, repo = value.split("/", 1)- return owner, repo- raise ValueError(f"Could not parse repository from '{value}'.")---def github_request(path: str, token: Optional[str], params: Optional[Dict[str, str]] = None) -> requests.Response:- headers = {- "Accept": "application/vnd.github+json",- "User-Agent": USER_AGENT,- "X-GitHub-Api-Version": GITHUB_API_VERSION,- }- if token:- headers["Authorization"] = f"Bearer {token}"- url = f"https://api.github.com/{path.lstrip('/')}"- response = requests.get(url, headers=headers, params=params or {})- if response.status_code == 401:- raise RuntimeError("GitHub authentication failed. Set the GITHUB_TOKEN environment variable.")- if response.status_code == 403 and "rate limit" in response.text.lower():- raise RuntimeError("GitHub rate limit exceeded. Provide a token to continue.")- return response---def fetch_repo(owner: str, repo: str, token: Optional[str]) -> Dict:- response = github_request(f"repos/{owner}/{repo}", token, params={"per_page": 1})- if response.status_code != 200:- raise RuntimeError(f"Failed to fetch repo metadata ({response.status_code}): {response.text}")- return response.json()---def fetch_readme_excerpt(owner: str, repo: str, token: Optional[str]) -> Optional[str]:- response = github_request(f"repos/{owner}/{repo}/readme", token)- if response.status_code != 200:- return None- payload = response.json()- content = payload.get("content")- if not content:- return None- try:- decoded = base64.b64decode(content).decode("utf-8", errors="replace")- except (ValueError, UnicodeDecodeError):- return None- for line in decoded.splitlines():- stripped = line.strip()- if not stripped:- continue- if stripped.startswith("#"):- stripped = stripped.lstrip("#").strip()- stripped = stripped.rstrip(".")- if stripped:- return stripped- return None---def normalize_project_name(repo_name: str) -> str:- if not repo_name:- return ""- if any(ch.isupper() for ch in repo_name if ch.isalpha()):- return repo_name- tokens = [token for token in re.split(r"[-_]", repo_name) if token]- if tokens:- return " ".join(token.capitalize() for token in tokens)- return repo_name.capitalize()---def iso_to_mmddyyyy(value: Optional[str]) -> str:- if not value:- return ""- try:- return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").strftime("%m/%d/%Y")- except ValueError:- return ""---def keyword_hits(keywords: Dict[str, str], sources: Sequence[str]) -> Set[str]:- if not sources:- return set()- haystack = " ".join(filter(None, sources)).lower()- matches: Set[str] = set()- for needle, mapped in keywords.items():- if re.search(rf"\b{re.escape(needle)}\b", haystack):- matches.add(mapped)- return matches---def infer_platforms(repo: Dict, ref: ReferenceData) -> Tuple[Set[str], List[str]]:- notes: List[str] = []- notes.append(- "Platforms were not inferred automatically.\n"- f"{available_text('platform', ref.platform_labels)}"- )- return set(), notes---def infer_tags(repo: Dict, ref: ReferenceData) -> Tuple[Set[str], List[str]]:- return set(), []---def infer_category(repo: Dict, ref: ReferenceData) -> Tuple[str, List[str]]:- return "", [- "Category must be specified manually.\n"- f"{available_text('category', ref.category_labels)}"- ]---def prompt_list(field_label: str, options: Dict[str, str], allow_empty: bool = False) -> List[str]:- valid_values = set(options.keys())- print(f"\n{available_text(field_label, options)}")- skip_hint = " (press Enter to skip)" if allow_empty else ""- prompt = f"Enter {field_label} ids (comma separated){skip_hint}:\n> "- while True:- raw = input(prompt).strip()- if not raw and allow_empty:- return []- values = [val.strip() for val in raw.split(",") if val.strip()]- invalid = [val for val in values if val not in valid_values]- if invalid:- print(f"Invalid values: {', '.join(invalid)}. Please try again.")- continue- if not values:- print("At least one value is required. Press Ctrl+C to abort.")- continue- return values---def prompt_value(field_label: str, options: Dict[str, str]) -> str:- valid_values = set(options.keys())- print(f"\n{available_text(field_label, options)}")- prompt = f"Enter {field_label} id:\n> "- while True:- raw = input(prompt).strip()- if raw in valid_values:- return raw- print(f"{raw} is not a valid value. Please try again.")---def fill_missing_with_input(entry: Dict, ref: ReferenceData) -> Dict:- if not sys.stdin.isatty():- return entry-- updated = entry.copy()- if not updated.get("platforms"):- updated["platforms"] = prompt_list("platform", ref.platform_labels)- if not updated.get("tags"):- updated["tags"] = prompt_list("tag", ref.tag_labels, allow_empty=True)- if not updated.get("category"):- updated["category"] = prompt_value("category", ref.category_labels)- return updated---def filter_resolved_notes(notes: List[str], entry: Dict) -> List[str]:- filtered: List[str] = []- for note in notes:- lowered = note.lower()- if "platform" in lowered and entry.get("platforms"):- continue- if "tag" in lowered and entry.get("tags"):- continue- if "category" in lowered and entry.get("category"):- continue- filtered.append(note)- return filtered---def build_entry(- repo_url: str,- repo_data: Dict,- ref: ReferenceData,- owner: str,- repo: str,- full_details: bool,- token: Optional[str],-) -> Tuple[Dict, List[str]]:- notes: List[str] = []- flags: Set[str] = set()-- repo_description = repo_data.get("description") or ""- readme_description: Optional[str] = None- if full_details and not repo_description:- readme_description = fetch_readme_excerpt(owner, repo, token)-- name = normalize_project_name(repo_data.get("name", ""))- platforms, platform_notes = infer_platforms(repo_data, ref)- notes.extend(platform_notes)- tags, tag_notes = infer_tags(repo_data, ref)- notes.extend(tag_notes)- category, category_notes = infer_category(repo_data, ref)- notes.extend(category_notes)-- entry = {- "name": name,- "description": "",- "repo_url": repo_url,- "tags": sorted(tags),- "platforms": sorted(platforms),- "category": category,- "stars": 0,- "flags": sorted(flags),- "last_commit": "",- "language": "",- "license": "",- "homepage_url": "",- }-- if full_details:- description_value = repo_description or readme_description or ""- entry["description"] = description_value- if readme_description and not repo_description:- flags.add("custom-description")- notes.append("Description pulled from README (custom-description flag added).")- elif not description_value:- notes.append("Repository has no description; field left blank.")-- entry["stars"] = repo_data.get("stargazers_count", 0)- entry["last_commit"] = iso_to_mmddyyyy(repo_data.get("pushed_at"))- entry["language"] = repo_data.get("language") or ""- license_data = repo_data.get("license") or {}- entry["license"] = license_data.get("spdx_id") or license_data.get("name") or ""- entry["homepage_url"] = repo_data.get("homepage") or ""-- entry["flags"] = sorted(flags)-- invalid_tags = [tag for tag in entry["tags"] if tag not in ref.tag_ids]- if invalid_tags:- raise ValueError(f"Invalid tag ids supplied: {', '.join(invalid_tags)}")-- invalid_platforms = [platform for platform in entry["platforms"] if platform not in ref.platform_ids]- if invalid_platforms:- raise ValueError(f"Invalid platform ids supplied: {', '.join(invalid_platforms)}")-- if entry["category"] and entry["category"] not in ref.category_ids:- raise ValueError(f"Invalid category id supplied: {entry['category']}")-- if not entry["platforms"]:- notes.append(- "No platforms detected; please review `platforms`.\n"- f"{available_text('platform', ref.platform_labels)}"- )- if not entry["category"]:- notes.append(- "Category missing; update `category` manually.\n"- f"{available_text('category', ref.category_labels)}"- )-- return entry, notes---def main() -> None:- parser = argparse.ArgumentParser(- description="Generate an applications.json entry from a GitHub repository URL."- )- parser.add_argument("repo", help="GitHub repository URL or owner/repo slug.")- parser.add_argument(- "--applications-file",- default=str(APPLICATIONS_FILE),- help="Path to applications.json (default: core/data/dynamic/applications.json).",- )- parser.add_argument(- "--full-details",- action="store_true",- help="Populate optional fields (description, stats, license, homepage) using GitHub data.",- )-- args = parser.parse_args()- owner, repo_name = parse_repo_identifier(args.repo)- repo_url = f"https://github.com/{owner}/{repo_name}"-- token = os.getenv("GITHUB_TOKEN")-- repo_data = fetch_repo(owner, repo_name, token)- ref = load_reference_data()- entry, notes = build_entry(repo_url, repo_data, ref, owner, repo_name, args.full_details, token)- entry = fill_missing_with_input(entry, ref)- notes = filter_resolved_notes(notes, entry)-- applications_path = Path(args.applications_file).resolve()-- print(json.dumps(entry, indent=4))- if notes:- print("\nNotes:")- for note in notes:- print(f"- {note}")-- try:- append_application(entry, applications_path)- except DuplicateRepositoryError as exc:- print(f"\nEntry skipped: {exc}")- print("Hint: If you meant to update that entry, edit applications.json directly.")- return-- print(f"\nAdded entry to {applications_path}")---if __name__ == "__main__":- main()diff --git a/scripts/utils/json_mod.py b/scripts/utils/json_mod.pydeleted file mode 100644index f26f6bb..0000000--- a/scripts/utils/json_mod.py+++ /dev/null@@ -1,62 +0,0 @@-import json--"""--# Load the JSON data from file-with open("core/data/dynamic/applications.json", "r", encoding="utf-8") as file:- data = json.load(file)--# Convert all platform entries to lowercase-for app in data.get("applications", []):- if "platforms" in app and isinstance(app["platforms"], list):- app["platforms"] = [platform.lower() for platform in app["platforms"]]--# Write the modified data back to the file-with open("core/data/dynamic/applications.json", "w", encoding="utf-8") as file:- json.dump(data, file, indent=4)--print("All platform entries have been converted to lowercase.")-"""--"""-# Load the JSON file-with open("core/data/dynamic/applications.json", "r", encoding="utf-8") as file:- data = json.load(file)--# Add "flags" and "stars" properties to each application-for app in data.get("applications", []):- app["homepage_url"] = ""--# Save the updated JSON back to the file-with open("core/data/dynamic/applications.json", "w", encoding="utf-8") as file:- json.dump(data, file, indent=4, ensure_ascii=False)--print("Operation successful: applications.json updated")-"""--# Load applications.json-with open("core/data/dynamic/applications.json", "r", encoding="utf-8") as f:- data = json.load(f)--# Platforms to check for-move_to_tags = {"cli", "cli-plus", "tui", "pip"}--for app in data.get("applications", []):- platforms = app.get("platforms", [])- tags = set(app.get("tags", []))-- # Move matching items to tags- new_platforms = []- for p in platforms:- if p in move_to_tags:- tags.add(p)- else:- new_platforms.append(p)-- # Update the application- app["platforms"] = new_platforms- app["tags"] = sorted(tags)--# Save the updated file-with open("core/data/dynamic/applications.json", "w", encoding="utf-8") as f:- json.dump(data, f, indent=4)diff --git a/scripts/utils/md_to_json.py b/scripts/utils/md_to_json.pydeleted file mode 100644index 74bf673..0000000--- a/scripts/utils/md_to_json.py+++ /dev/null@@ -1,48 +0,0 @@-import json-import re---def parse_readme(readme_path):- with open(readme_path, 'r', encoding='utf-8') as file:- lines = file.readlines()-- applications = []- category = None-- for line in lines:- line = line.strip()-- # Match category headers- category_match = re.match(r"^### (.+)", line)- if category_match:- category = category_match.group(1).lower().replace(" ", "-")- continue-- # Match application entries with optional tag- app_match = re.match(r"\| \[(.+)\]\((https://github.com/[^)]+)\)(?: `([^`]+)`)? \| (.+?) \| (.+?) \|", line)- if app_match and category:- name, link, tag, description, platforms = app_match.groups()- applications.append({- "name": name,- "description": description,- "repo_url": link,- "tags": [tag] if tag else [],- "platforms": platforms.split(),- "category": category- })-- return applications---def save_to_json(data, output_path):- with open(output_path, 'w', encoding='utf-8') as file:- json.dump(data, file, indent=4, ensure_ascii=False)---if __name__ == "__main__":- readme_path = "README.md" # Update with actual path- output_path = "applications.json"-- parsed_data = parse_readme(readme_path)- save_to_json(parsed_data, output_path)- print(f"Converted README to {output_path}")