Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
9.6 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Validate search-from-T2 gold answers against repo ground truth.

Checks that gold files exist in the repo and gold functions resolve to
real definitions. Reuses validation logic from validate_gold_answers.py.

Also auto-populates alt_functions from class methods for method-level flexibility.

Ported from craft-bench scripts/search/validate_t2_gold.py.
"""

from __future__ import annotations

import json
import os
import sys
from collections import defaultdict

from craft_taskgen.mining.repo_indexer import RepoIndexer
from craft_taskgen.mining.schemas import RepoGroundTruth

# ---------------------------------------------------------------------------
# Ground truth helpers (simplified from validate_gold_answers.py)
# ---------------------------------------------------------------------------


def build_file_set(gt: RepoGroundTruth) -> set[str]:
    """All files in the repo, normalized."""
    return {f.lstrip("./") for f in gt.file_structure.files}


def build_class_methods(gt: RepoGroundTruth) -> dict[str, list[str]]:
    """Build {qualified_class_name: [method_name, ...]} from ground truth.

    Keys are like 'uvicorn.config.Config', values are like
    ['get_loop_factory', 'setup_event_loop', 'load', '__init__'].
    """
    result: dict[str, list[str]] = defaultdict(list)
    for cls in gt.type_defs.classes:
        # cls.name is just 'Config', need to qualify with module
        module = cls.file_path.replace("/", ".").removesuffix(".py")
        qualified = f"{module}.{cls.name}"
        for method in cls.methods:
            method_name = method.name if hasattr(method, "name") else str(method)
            result[qualified].append(method_name)
    return dict(result)


def build_all_callees(gt: RepoGroundTruth) -> set[str]:
    """All callee names from call graph edges."""
    return {edge.callee for edge in gt.call_graph.edges}


def check_file(filepath: str, file_set: set[str]) -> str | None:
    """Return error message if file doesn't exist, else None."""
    normalized = filepath.lstrip("./").rstrip("/")
    if normalized in file_set:
        return None
    # Try without leading directory
    for f in file_set:
        if f.endswith("/" + normalized) or f == normalized:
            return None
    return f"file not in repo: {filepath}"


def check_function(func_name: str, class_methods: dict[str, list[str]], callees: set[str]) -> str | None:
    """Return error message if function can't be resolved, else None."""
    # Direct callee match
    if func_name in callees:
        return None

    # Parse into module.Class.method
    parts = func_name.split(".")
    if len(parts) >= 3:
        # Try class.method match
        method = parts[-1]
        class_name = parts[-2]
        # Check if any qualified class has this method
        for qual_class, methods in class_methods.items():
            if qual_class.endswith(f".{class_name}") and method in methods:
                return None

    # Try leaf name in callees
    leaf = parts[-1] if parts else func_name
    if leaf in callees:
        return None

    # Try tail match (Class.method)
    if len(parts) >= 2:
        tail = f"{parts[-2]}.{parts[-1]}"
        for callee in callees:
            if callee.endswith(tail):
                return None

    return f"function not resolvable: {func_name}"


# ---------------------------------------------------------------------------
# Alt-function expansion
# ---------------------------------------------------------------------------


def expand_alt_functions(
    gold_functions: list[str],
    class_methods: dict[str, list[str]],
) -> list[str]:
    """For each gold function module.Class.method, add other methods on the same class."""
    alt = []
    seen = set(gold_functions)

    for func in gold_functions:
        parts = func.split(".")
        if len(parts) < 3:
            continue

        class_name = parts[-2]

        # Find the class in ground truth and add its other methods
        for qual_class, methods in class_methods.items():
            if qual_class.endswith(f".{class_name}"):
                module = qual_class.rsplit(".", 1)[0]
                for m in methods:
                    qualified = f"{module}.{class_name}.{m}"
                    if qualified not in seen:
                        alt.append(qualified)
                        seen.add(qualified)
                break

    return alt


# ---------------------------------------------------------------------------
# Ground truth loading
# ---------------------------------------------------------------------------


def load_ground_truths(repos_dir: str, contexts_path: str) -> dict[str, RepoGroundTruth]:
    """Load ground truth for each unique repo referenced by contexts."""
    with open(contexts_path) as f:
        contexts = json.load(f)

    gts: dict[str, RepoGroundTruth] = {}
    for ctx in contexts:
        url = ctx["solve_info"]["upstream_url"]
        repo_name = url.rstrip("/").removesuffix(".git").split("/")[-1] if url else ""
        if repo_name in gts:
            continue
        repo_dir = os.path.join(repos_dir, repo_name)
        if not os.path.isdir(repo_dir):
            print(f"  WARNING: repo not found: {repo_dir}", file=sys.stderr)
            continue
        print(f"  Indexing {repo_name}...", file=sys.stderr)
        indexer = RepoIndexer(repo_dir)
        gts[repo_name] = indexer.index()

    return gts


# ---------------------------------------------------------------------------
# Top-level entry point (called by step_validate)
# ---------------------------------------------------------------------------


def run_validate(
    contexts_dir: str,
    repos_dir: str = "repos",
    fix_alt_funcs: bool = True,
) -> None:
    """Validate gold answers against repo AST and optionally expand alt_functions.

    Parameters
    ----------
    contexts_dir:
        Directory containing _all_contexts.json and approach-{a,b,c}/search_tasks.json.
    repos_dir:
        Directory containing cloned repos.
    fix_alt_funcs:
        If True, auto-populate alt_functions from class methods.
    """
    contexts_path = os.path.join(contexts_dir, "_all_contexts.json")
    with open(contexts_path) as f:
        contexts = {c["task_id"]: c for c in json.load(f)}

    # Build ground truth for each repo
    print("Loading ground truth...", file=sys.stderr)
    gts = load_ground_truths(repos_dir, contexts_path)

    # Build indexes per repo
    repo_indexes: dict[str, tuple[set[str], dict[str, list[str]], set[str]]] = {}
    for repo_name, gt in gts.items():
        file_set = build_file_set(gt)
        class_methods = build_class_methods(gt)
        callees = build_all_callees(gt)
        repo_indexes[repo_name] = (file_set, class_methods, callees)
        print(f"  {repo_name}: {len(file_set)} files, {len(class_methods)} classes, {len(callees)} callees")

    # Validate all tasks
    print("\nValidating tasks...\n")

    all_tasks = []
    for approach in ["a", "b", "c"]:
        path = os.path.join(contexts_dir, f"approach-{approach}", "search_tasks.json")
        if not os.path.exists(path):
            continue
        with open(path) as f:
            tasks = json.load(f)
        all_tasks.extend([(approach, t, path) for t in tasks])

    total = len(all_tasks)
    good = 0
    file_errors = 0
    func_errors = 0
    alt_added = 0

    for approach, task, tasks_path in sorted(all_tasks, key=lambda x: x[1]["id"]):
        gold = task["gold_answer"]
        parent = task.get("parent_t2_task", "")
        ctx = contexts.get(parent, {})
        url = ctx.get("solve_info", {}).get("upstream_url", "")
        repo_name = url.rstrip("/").removesuffix(".git").split("/")[-1] if url else ""

        if repo_name not in repo_indexes:
            print(f"  [{approach.upper()}] {task['id']}: SKIP (no ground truth for {repo_name})")
            continue

        file_set, class_methods, callees = repo_indexes[repo_name]
        issues = []

        # Check files
        for f in gold.get("files", []):
            err = check_file(f, file_set)
            if err:
                issues.append(f"FILE: {err}")
                file_errors += 1

        # Check functions
        for fn in gold.get("functions", []):
            err = check_function(fn, class_methods, callees)
            if err:
                issues.append(f"FUNC: {err}")
                func_errors += 1

        # Expand alt_functions if requested
        if fix_alt_funcs:
            new_alts = expand_alt_functions(gold.get("functions", []), class_methods)
            existing_alts = set(gold.get("alt_functions", []))
            added = [a for a in new_alts if a not in existing_alts]
            if added:
                gold["alt_functions"] = list(existing_alts | set(new_alts))
                alt_added += len(added)

        if issues:
            print(f"  [{approach.upper()}] {task['id']} ({parent}):")
            for issue in issues:
                print(f"    {issue}")
        else:
            good += 1

    print(f"\n{'=' * 60}")
    print(f"Total: {total}  Good: {good}  File errors: {file_errors}  Func errors: {func_errors}")
    if fix_alt_funcs:
        print(f"Alt functions added: {alt_added}")

    # Write back if we modified tasks
    if fix_alt_funcs and alt_added > 0:
        # Group by approach and rewrite
        by_path: dict[str, list] = defaultdict(list)
        for approach, task, tasks_path in all_tasks:
            by_path[tasks_path].append(task)
        for path, tasks in by_path.items():
            with open(path, "w") as f:
                json.dump(tasks, f, indent=2)
            print(f"  Updated {path}")