Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
3.3 KB
Raw Blame
#!/usr/bin/env python3
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Dedup a craft-repos-style CSV and remove any repos that appear in swebench-repos.csv.

Usage:
    python3 scripts/dedup_repos.py references/repo_list_v2.csv
    python3 scripts/dedup_repos.py references/repo_list_v2.csv --out references/repo_list_v2-clean.csv
    python3 scripts/dedup_repos.py references/repo_list_v2.csv --swebench references/swebench-repos.csv
"""

from __future__ import annotations

import argparse
import csv
import sys
from pathlib import Path

DEFAULT_EXCLUDED = Path("references/excluded-repos.csv")
FIELDS = ["short_name", "github_repo", "github_url", "stars", "license", "domain", "description"]


def run_dedup(rows: list[dict], excluded_path: Path) -> tuple[list[dict], list[dict], list[dict]]:
    """Return (kept, duplicates_removed, excluded_removed)."""
    seen: set[str] = set()
    deduped, duplicates = [], []
    for r in rows:
        key = r["github_repo"]
        if key in seen:
            duplicates.append(r)
        else:
            seen.add(key)
            deduped.append(r)

    excluded: set[str] = set()
    if excluded_path.exists():
        with excluded_path.open() as f:
            excluded = {r["github_repo"] for r in csv.DictReader(f)}

    kept = [r for r in deduped if r["github_repo"] not in excluded]
    excluded_removed = [r for r in deduped if r["github_repo"] in excluded]
    return kept, duplicates, excluded_removed


def main() -> None:
    parser = argparse.ArgumentParser(description="Dedup repo CSV and remove swebench repos.")
    parser.add_argument("csv_file", type=Path, help="Input CSV (craft-repos-style)")
    parser.add_argument("--out", type=Path, default=None, help="Output path (default: overwrites input)")
    parser.add_argument(
        "--exclude",
        type=Path,
        default=DEFAULT_EXCLUDED,
        help="Excluded repos CSV (default: references/excluded-repos.csv)",
    )
    args = parser.parse_args()

    if not args.csv_file.exists():
        sys.exit(f"ERROR: input file not found: {args.csv_file}")

    with args.csv_file.open() as f:
        rows = list(csv.DictReader(f))

    if not rows:
        sys.exit(f"ERROR: {args.csv_file} is empty or contains no data rows.")

    kept, duplicates, excluded_removed = run_dedup(rows, args.exclude)

    out_file = args.out or args.csv_file
    tmp = out_file.with_suffix(".tmp")
    try:
        with open(tmp, "w", newline="") as f:
            writer = csv.DictWriter(f, fieldnames=FIELDS)
            writer.writeheader()
            writer.writerows(kept)
        tmp.rename(out_file)
    except OSError as e:
        tmp.unlink(missing_ok=True)
        sys.exit(f"ERROR: failed to write {out_file}: {e}")

    print(f"Input:            {len(rows)}")
    if duplicates:
        print(f"Duplicates removed: -{len(duplicates)}  ({[r['short_name'] for r in duplicates]})")
    else:
        print("Duplicates removed:  0")
    if excluded_removed:
        names = [r["short_name"] for r in excluded_removed]
        print(f"Excluded removed:   -{len(excluded_removed)}  ({names})")
    else:
        print("Excluded removed:    0")
    print(f"Final:             {len(kept)}  → {out_file}")


if __name__ == "__main__":
    main()