Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
6.9 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Deduplicate search-from-T2 tasks by embedding-based instruction similarity.

Loads tasks from all approaches, embeds instructions, removes near-duplicates
(cosine >= threshold), and merges removed tasks' gold into kept tasks' alt fields.
"""

from __future__ import annotations

import json
import math
import os


def _cosine_similarity(a: list[float], b: list[float]) -> float:
    dot = sum(x * y for x, y in zip(a, b))
    norm_a = math.sqrt(sum(x * x for x in a))
    norm_b = math.sqrt(sum(x * x for x in b))
    if norm_a == 0 or norm_b == 0:
        return 0.0
    return dot / (norm_a * norm_b)


def _gold_richness(task: dict) -> tuple[int, int]:
    gold = task.get("gold_answer", {})
    n_elements = len(gold.get("files", [])) + len(gold.get("functions", []))
    n_assertions = len(gold.get("assertions", []))
    return (n_elements, n_assertions)


def _merge_alt_gold(kept: dict, removed: dict) -> None:
    """Merge removed task's gold files/functions into kept's alt fields. Mutates kept."""
    kept_gold = kept.get("gold_answer", {})
    removed_gold = removed.get("gold_answer", {})

    existing_files = set(kept_gold.get("files", [])) | set(kept_gold.get("alt_files", []))
    novel_files = [f for f in removed_gold.get("files", []) if f not in existing_files]

    existing_funcs = set(kept_gold.get("functions", [])) | set(kept_gold.get("alt_functions", []))
    novel_funcs = [f for f in removed_gold.get("functions", []) if f not in existing_funcs]

    if novel_files:
        kept_gold.setdefault("alt_files", []).extend(novel_files)
    if novel_funcs:
        kept_gold.setdefault("alt_functions", []).extend(novel_funcs)


def embed_instructions(instructions: list[str], model: str) -> list[list[float]]:
    """Batch-embed instructions via litellm."""
    import litellm

    response = litellm.embedding(model=model, input=instructions)
    return [item["embedding"] for item in response.data]


def deduplicate(
    tasks: list[dict],
    embeddings: list[list[float]],
    threshold: float = 0.65,
) -> tuple[list[dict], list[dict]]:
    """Remove near-duplicate tasks by cosine similarity on instruction embeddings.

    Keeps the task with richer gold (more files+functions, then more assertions).
    Merges removed task's gold into kept task's alt_files/alt_functions.

    Returns (kept_tasks, dedup_pairs).
    """
    n = len(tasks)
    removed: set[int] = set()
    dedup_pairs: list[dict] = []
    richness = [_gold_richness(t) for t in tasks]

    for i in range(n):
        if i in removed:
            continue
        for j in range(i + 1, n):
            if j in removed:
                continue
            sim = _cosine_similarity(embeddings[i], embeddings[j])
            if sim >= threshold:
                if richness[j] > richness[i]:
                    kept_idx, removed_idx = j, i
                else:
                    kept_idx, removed_idx = i, j

                _merge_alt_gold(tasks[kept_idx], tasks[removed_idx])

                dedup_pairs.append(
                    {
                        "cosine": round(sim, 4),
                        "kept": tasks[kept_idx]["id"],
                        "removed": tasks[removed_idx]["id"],
                        "kept_richness": richness[kept_idx],
                        "removed_richness": richness[removed_idx],
                    }
                )
                removed.add(removed_idx)
                if removed_idx == i:
                    break

    kept = [t for idx, t in enumerate(tasks) if idx not in removed]
    return kept, sorted(dedup_pairs, key=lambda p: -p["cosine"])


def run_dedup(
    output_dir: str,
    *,
    threshold: float = 0.65,
    embedding_model: str = "openai/azure/openai/text-embedding-3-small",
) -> None:
    """Load tasks from all approaches, embed, deduplicate, and write back.

    Args:
        output_dir: Directory containing approach-{a,b,c}/search_tasks.json.
        threshold: Cosine similarity threshold for duplicate detection.
        embedding_model: LiteLLM model identifier for embedding.
    """
    # Load all tasks across approaches, tracking which approach each came from
    all_tasks: list[dict] = []
    approach_map: dict[str, str] = {}  # task_id -> approach letter
    for approach in ["a", "b", "c"]:
        path = os.path.join(output_dir, f"approach-{approach}", "search_tasks.json")
        if not os.path.exists(path):
            continue
        with open(path) as f:
            tasks = json.load(f)
        for t in tasks:
            approach_map[t["id"]] = approach
        all_tasks.extend(tasks)
        print(f"  Approach {approach.upper()}: {len(tasks)} tasks")

    print(f"  Total: {len(all_tasks)} tasks")

    if not all_tasks:
        print("No tasks found.")
        return

    # Embed
    print(f"\nEmbedding {len(all_tasks)} instructions (model: {embedding_model})...")
    instructions = [t.get("instruction", "") for t in all_tasks]
    embeddings = embed_instructions(instructions, embedding_model)
    print(f"  Got {len(embeddings)} embeddings ({len(embeddings[0])} dims)")

    # Dedup
    print(f"\nDeduplicating (threshold: {threshold})...")
    kept, pairs = deduplicate(all_tasks, embeddings, threshold=threshold)
    n_removed = len(all_tasks) - len(kept)

    print(f"  Removed: {n_removed} duplicates ({len(pairs)} pairs)")
    print(f"  Kept: {len(kept)} tasks")

    # Show top pairs
    if pairs:
        print(f"\n  Top duplicate pairs (cosine >= {threshold}):")
        for p in pairs[:15]:
            print(f"    {p['cosine']:.3f}  kept={p['kept']}  removed={p['removed']}")
        if len(pairs) > 15:
            print(f"    ... and {len(pairs) - 15} more")

    # Per-approach breakdown
    for approach in ["a", "b", "c"]:
        before = sum(1 for _tid, a in approach_map.items() if a == approach)
        after = sum(1 for t in kept if approach_map.get(t["id"]) == approach)
        print(f"  Approach {approach.upper()}: {before} -> {after} (-{before - after})")

    # Write back to approach files
    for approach in ["a", "b", "c"]:
        path = os.path.join(output_dir, f"approach-{approach}", "search_tasks.json")
        if not os.path.exists(path):
            continue
        approach_kept = [t for t in kept if approach_map.get(t["id"]) == approach]
        with open(path, "w") as f:
            json.dump(approach_kept, f, indent=2)
        print(f"  Updated {path} ({len(approach_kept)} tasks)")

    # Write manifest
    manifest = {
        "threshold": threshold,
        "embedding_model": embedding_model,
        "total_before": len(all_tasks),
        "total_after": len(kept),
        "duplicates_removed": n_removed,
        "pairs": pairs,
    }
    manifest_path = os.path.join(output_dir, "dedup_manifest.json")
    with open(manifest_path, "w") as f:
        json.dump(manifest, f, indent=2)
    print(f"  Manifest: {manifest_path}")