Skip to content
Python
24.9 KB
Raw Blame
"""
Compare SVG ball path trajectories with ground truth.
Processes multiple models from batch directories and computes distance metrics.

Usage:
    python3 compare_lines.py
    python3 compare_lines.py --n_points 200
"""
import numpy as np
from svgpathtools import parse_path
import xml.etree.ElementTree as ET
import json
import matplotlib
matplotlib.use('Agg')  # Use non-interactive backend
import matplotlib.pyplot as plt
def perturb_degenerate_segment(segment, epsilon=0.1):
    """
    Add tiny perturbations to bezier segments to avoid numerical issues.

    Always applies small perturbation to prevent degenerate cases that cause
    arc-length computation to fail.

    Parameters:
    -----------
    segment : bezier curve segment
        A segment from svgpathtools
    epsilon : float
        Small value to add (in pixels, scaled to SVG space)

    Returns:
    --------
    segment : slightly perturbed segment
    """
    from svgpathtools import Line, QuadraticBezier, CubicBezier, Arc
    import numpy as np

    # Always perturb to avoid edge cases
    if isinstance(segment, Line):
        start, end = segment.start, segment.end
        # Add small perturbation to avoid perfectly vertical/horizontal lines
        end = end + complex(epsilon * 0.1, epsilon * 0.1)
        return Line(start, end)

    elif isinstance(segment, QuadraticBezier):
        start, control, end = segment.start, segment.control, segment.end
        # Perturb control point to ensure non-degeneracy
        control = control + complex(epsilon, -epsilon * 0.5)
        return QuadraticBezier(start, control, end)

    elif isinstance(segment, CubicBezier):
        start, c1, c2, end = segment.start, segment.control1, segment.control2, segment.end
        # Perturb both control points
        c1 = c1 + complex(epsilon * 0.7, -epsilon * 0.3)
        c2 = c2 + complex(-epsilon * 0.3, epsilon * 0.7)
        return CubicBezier(start, c1, c2, end)

    # Return Arc and other types unmodified
    return segment

def extract_points_from_svg(svg_path, n_points=500, target_size=512):
    """
    Extract n_points uniformly sampled along the entire SVG path.

    Scales coordinates from SVG canvas space to target_size x target_size space
    to match ground truth coordinate system.

    Parameters:
    -----------
    svg_path : str
        Path to the SVG file
    n_points : int
        Total number of points to sample uniformly along entire trajectory
    target_size : int
        Target canvas size (e.g., 512 for 512x512 ground truth space)

    Returns:
    --------
    np.ndarray
        Array of shape (n_points, 2) containing [x, y] coordinates scaled to target space
    """
    import xml.etree.ElementTree as ET
    from svgpathtools import parse_path, Path
    import numpy as np

    # Parse the SVG file
    tree = ET.parse(svg_path)
    root = tree.getroot()

    # Get SVG canvas dimensions for scaling
    svg_width = float(root.get('width', 2091))
    svg_height = float(root.get('height', 2091))

    # Calculate scale factors to convert to target space (e.g., 512x512)
    scale_x = target_size / svg_width
    scale_y = target_size / svg_height

    # Handle XML namespaces
    namespaces = {'svg': 'http://www.w3.org/2000/svg'}
    paths = root.findall('.//path') + root.findall('.//svg:path', namespaces)

    if not paths:
        raise ValueError("No path elements found in SVG")

    # Parse and concatenate all paths into one continuous path
    all_segments = []
    for path_elem in paths:
        d = path_elem.get('d')
        if d:
            path = parse_path(d)
            # Perturb degenerate segments (use larger epsilon to avoid numerical issues)
            for seg in path:
                perturbed_seg = perturb_degenerate_segment(seg, epsilon=0.1)
                all_segments.append(perturbed_seg)

    if not all_segments:
        raise ValueError("No valid path data found")

    # Create one continuous path from all segments
    continuous_path = Path(*all_segments)

    # Try arc-length sampling first, fall back to parameter sampling if it fails
    try:
        # Get total length
        total_length = continuous_path.length()

        # Check for NaN or inf length
        if not np.isfinite(total_length) or total_length == 0:
            raise ValueError("Invalid path length")

        # Sample uniformly by arc length
        points = []
        for i in range(n_points):
            # Distance along the path
            distance = (i / (n_points - 1)) * total_length if n_points > 1 else 0

            # Convert distance to parameter t
            t = continuous_path.ilength(distance)

            # Get point at parameter t
            pt = continuous_path.point(t)

            # Scale to target coordinate space
            scaled_x = pt.real * scale_x
            scaled_y = pt.imag * scale_y
            points.append([scaled_x, scaled_y])

        return np.array(points)

    except Exception as e:
        # Fallback: sample uniformly by parameter t (not arc length)
        # This is less accurate but works for degenerate paths
        import warnings
        warnings.warn(f"Arc-length sampling failed ({str(e)[:50]}), using parameter-based sampling")

        points = []
        for i in range(n_points):
            # Sample parameter t uniformly from 0 to 1
            t = i / (n_points - 1) if n_points > 1 else 0

            # Get point at parameter t
            pt = continuous_path.point(t)

            # Scale to target coordinate space
            scaled_x = pt.real * scale_x
            scaled_y = pt.imag * scale_y
            points.append([scaled_x, scaled_y])

        return np.array(points)

def extract_points_from_gt_json(json_path, n_points=100, ball_body_index=None):
    """
    Extract n_points uniformly sampled by arc-length from ground truth trajectory JSON.

    Samples uniformly along the physical path distance (arc length), not by time/index.
    This matches the SVG sampling method for fair comparison.

    Transforms coordinates from physics coordinate system (origin bottom-left, Y up)
    to SVG coordinate system (origin top-left, Y down).

    Parameters:
    -----------
    json_path : str
        Path to the JSON file containing trajectories
    n_points : int
        Number of points to sample uniformly along arc length
    ball_body_index : str or None
        The key in the trajectories dict for the ball. If None, automatically finds the ball.

    Returns:
    --------
    np.ndarray
        Array of shape (n_points, 2) containing [x, y] coordinates in SVG space
    """
    # Load the JSON data
    with open(json_path, 'r') as f:
        data = json.load(f)

    # Get scene height for coordinate transformation
    scene_height = data['scene']['height']

    # Find the ball trajectory
    # If ball_body_index not specified, find it automatically
    if ball_body_index is None:
        trajectories = data.get('trajectories', {})
        if not trajectories:
            raise ValueError("No trajectories found in GT JSON")

        # Look for common ball body indices
        for possible_key in ['10', '9', '8', '11', '7']:
            if possible_key in trajectories:
                ball_body_index = possible_key
                break

        # If still not found, just take the first trajectory
        if ball_body_index is None:
            ball_body_index = list(trajectories.keys())[0]

    # Extract the trajectory points for the ball
    trajectory = data['trajectories'][ball_body_index]

    # Convert to numpy array and transform Y coordinates to SVG space
    # Physics: origin at bottom-left, Y increases upward
    # SVG: origin at top-left, Y increases downward
    # Transformation: svg_y = scene_height - physics_y
    all_points = np.array([[pt['x'], scene_height - pt['y']] for pt in trajectory])

    # Total number of points in ground truth
    total_points = len(all_points)

    if n_points >= total_points:
        # If requesting more points than available, return all points
        return all_points

    # Calculate cumulative arc length along the trajectory
    # distances[i] = distance between point i and point i+1
    distances = np.linalg.norm(np.diff(all_points, axis=0), axis=1)
    cumulative_distances = np.concatenate([[0], np.cumsum(distances)])
    total_length = cumulative_distances[-1]

    # Sample uniformly along arc length
    target_distances = np.linspace(0, total_length, n_points)

    # Interpolate to find points at target distances
    sampled_points = np.zeros((n_points, 2))
    for i, target_dist in enumerate(target_distances):
        # Find the segment containing this target distance
        idx = np.searchsorted(cumulative_distances, target_dist)

        if idx == 0:
            sampled_points[i] = all_points[0]
        elif idx >= len(all_points):
            sampled_points[i] = all_points[-1]
        else:
            # Linear interpolation between points idx-1 and idx
            seg_start_dist = cumulative_distances[idx - 1]
            seg_end_dist = cumulative_distances[idx]
            seg_length = seg_end_dist - seg_start_dist

            if seg_length > 0:
                # How far along this segment?
                t = (target_dist - seg_start_dist) / seg_length
                sampled_points[i] = (1 - t) * all_points[idx - 1] + t * all_points[idx]
            else:
                sampled_points[i] = all_points[idx - 1]

    return sampled_points

def visualize_trajectory(points, title="Sampled Trajectory", save_path=None):
    """Visualize the sampled trajectory points"""
    import matplotlib.pyplot as plt
    
    # fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 8))
    fig, ax1 = plt.subplots(figsize=(16, 8))
    
    # Plot 1: Full trajectory with all points
    ax1.plot(points[:, 0], points[:, 1], 'g-', linewidth=2, alpha=0.7, label='Trajectory')
    ax1.scatter(points[0, 0], points[0, 1], c='blue', s=200, zorder=5, 
                marker='o', edgecolors='black', linewidths=2, label='Start')
    ax1.scatter(points[-1, 0], points[-1, 1], c='red', s=200, zorder=5, 
                marker='s', edgecolors='black', linewidths=2, label='End')
    ax1.scatter(points[::1, 0], points[::1, 1], c='orange', s=50, zorder=4, 
                alpha=0.6, label='Sample points')
    ax1.invert_yaxis()  # SVG Y-axis points down
    ax1.axis('equal')
    ax1.set_title(f'{title}\n({len(points)} points total)', fontsize=14, fontweight='bold')
    ax1.set_xlabel('X', fontsize=12)
    ax1.set_ylabel('Y', fontsize=12)
    ax1.legend(loc='best', fontsize=10)
    ax1.grid(True, alpha=0.3)
    
    
    plt.tight_layout()
    
    # if save_path:
    #     plt.savefig(save_path, dpi=150, bbox_inches='tight')
    #     print(f"Saved visualization to {save_path}")
    
    plt.show()

def compute_average_min_distance(svg_points, gt_points):
    """
    Compute average minimum distance and MSE from SVG points to GT points.

    For each SVG point, finds the closest GT point and computes the distance.
    Then calculates both the average and MSE (Mean Squared Error) of these minimum distances.

    Parameters:
    -----------
    svg_points : np.ndarray
        Array of shape (n, 2) containing SVG trajectory points
    gt_points : np.ndarray
        Array of shape (m, 2) containing ground truth trajectory points

    Returns:
    --------
    tuple of (float, float)
        (average_min_distance, mse_min_distance)
    """
    min_distances = []

    for svg_pt in svg_points:
        # Compute distances from this SVG point to all GT points
        distances = np.linalg.norm(gt_points - svg_pt, axis=1)
        # Find the minimum distance
        min_dist = np.min(distances)
        min_distances.append(min_dist)

    # Convert to numpy array for easier computation
    min_distances = np.array(min_distances)

    # Average all minimum distances
    avg_min_distance = np.mean(min_distances)

    # MSE of minimum distances
    mse_min_distance = np.mean(min_distances ** 2)

    return avg_min_distance, mse_min_distance

def visualize_comparison(svg_points, gt_points, title="SVG vs Ground Truth Comparison", save_path=None, background_image_path=None):
    """
    Visualize both SVG trajectory and ground truth trajectory for comparison

    Parameters:
    -----------
    svg_points : np.ndarray
        Points sampled from SVG path
    gt_points : np.ndarray
        Points sampled from ground truth trajectory
    title : str
        Title for the plot
    save_path : str
        Optional path to save the figure
    background_image_path : str
        Optional path to background image to overlay trajectories on
    """
    import matplotlib.pyplot as plt
    from PIL import Image
    import os

    fig, ax = plt.subplots(figsize=(16, 10))

    # Display background image if provided
    if background_image_path and os.path.exists(background_image_path):
        try:
            img = Image.open(background_image_path)
            # Display image in 512x512 coordinate space (0,0 at top-left)
            ax.imshow(img, extent=[0, 512, 512, 0], aspect='auto', alpha=0.5, zorder=1)
        except Exception as e:
            print(f"Warning: Could not load background image: {e}")

    # Plot ground truth trajectory
    ax.plot(gt_points[:, 0], gt_points[:, 1], 'b-', linewidth=3, alpha=0.8, label='GT Trajectory', zorder=3)
    ax.scatter(gt_points[:, 0], gt_points[:, 1], c='blue', s=50, zorder=4, alpha=0.6)
    ax.scatter(gt_points[0, 0], gt_points[0, 1], c='blue', s=200, zorder=5,
                marker='o', edgecolors='black', linewidths=2)
    ax.scatter(gt_points[-1, 0], gt_points[-1, 1], c='darkblue', s=200, zorder=5,
                marker='s', edgecolors='black', linewidths=2)

    # Plot SVG trajectory
    ax.plot(svg_points[:, 0], svg_points[:, 1], 'g-', linewidth=3, alpha=0.8, label='SketchVLM', zorder=3)
    ax.scatter(svg_points[:, 0], svg_points[:, 1], c='green', s=50, zorder=4, alpha=0.6)
    ax.scatter(svg_points[0, 0], svg_points[0, 1], c='green', s=200, zorder=5,
                marker='o', edgecolors='black', linewidths=2)
    ax.scatter(svg_points[-1, 0], svg_points[-1, 1], c='darkgreen', s=200, zorder=5,
                marker='s', edgecolors='black', linewidths=2)

    # Set axis limits and properties
    ax.set_xlim(0, 512)
    ax.set_ylim(512, 0)  # Inverted Y-axis to match SVG coordinate system
    ax.axis('equal')
    ax.set_title(f'{title}', fontsize=14, fontweight='bold')
    ax.set_xlabel('X', fontsize=12)
    ax.set_ylabel('Y', fontsize=12)
    ax.legend(loc='upper center', fontsize=12, ncol=2)
    ax.grid(True, alpha=0.3)

    plt.tight_layout()

    if save_path:
        plt.savefig(save_path, dpi=150, bbox_inches='tight')
        print(f"Saved visualization to {save_path}")

    plt.close()

def process_all_svg_files(svg_dir, output_dir, n_points=100):
    """
    Process all SVG files in a directory, finding their GT pairs and computing metrics.

    Parameters:
    -----------
    svg_dir : str
        Directory containing SVG files and their corresponding JSON metadata
    output_dir : str
        Directory to save comparison visualizations and metrics
    n_points : int
        Number of points to sample from each trajectory

    Returns:
    --------
    list of dict
        List of results for each SVG-GT pair
    """
    import os
    import glob
    import csv

    # Create output directory if it doesn't exist
    os.makedirs(output_dir, exist_ok=True)

    # Find all SVG files
    svg_files = sorted(glob.glob(os.path.join(svg_dir, "*.svg")))
    print(f"Found {len(svg_files)} SVG files to process\n")

    results = []

    for svg_file in svg_files:
        # Get base name (e.g., "batch0_item_00000" or "item_00000")
        base_name_with_prefix = os.path.splitext(os.path.basename(svg_file))[0]

        # Strip batch prefix if present (e.g., "batch0_" or "batch1_")
        import re
        base_name = re.sub(r'^batch\d+_', '', base_name_with_prefix)

        json_file = os.path.join(svg_dir, f"{base_name_with_prefix}.json")

        # Read source_image from JSON to find GT file
        if not os.path.exists(json_file):
            print(f"Warning: No JSON file found for {base_name_with_prefix}, skipping...")
            continue

        try:
            with open(json_file, 'r') as f:
                metadata = json.load(f)

            source_image = metadata.get('source_image', '')
            if not source_image:
                print(f"Warning: No source_image in {base_name}.json, skipping...")
                continue

            # Extract GT identifier (e.g., "run_001_1" from "datasets/ball_path/run_001_1.png")
            # Normalize path separators (handle both / and \)
            source_image_normalized = source_image.replace('\\', '/')
            # Get just the filename without path or extension
            gt_identifier = os.path.splitext(os.path.basename(source_image_normalized))[0]

            # Find GT JSON file - check both batch directories
            gt_json_path = None
            for gt_base_dir in ["/Users/log/Github/sketchvlm/datasets/large_run_split",
                               "/Users/log/Github/sketchvlm/datasets/large_second_batch"]:
                potential_path = os.path.join(gt_base_dir, gt_identifier, "random_scene_metadata.json")
                if os.path.exists(potential_path):
                    gt_json_path = potential_path
                    break

            if not gt_json_path:
                print(f"Warning: GT file not found for {gt_identifier}, skipping...")
                continue

            print(f"Processing {base_name}...")
            print(f"  SVG: {svg_file}")
            print(f"  GT:  {gt_json_path}")

            # Extract points
            svg_points = extract_points_from_svg(svg_file, n_points=n_points, target_size=512)
            gt_points = extract_points_from_gt_json(gt_json_path, n_points=n_points)

            # Compute metrics
            avg_min_dist, mse_min_dist = compute_average_min_distance(svg_points, gt_points)

            print(f"  Average min distance: {avg_min_dist:.2f} pixels")
            print(f"  MSE min distance: {mse_min_dist:.2f} pixels²")

            # Build path to source image (use normalized path)
            source_image_path = os.path.join("/Users/log/Github/sketchvlm", source_image_normalized)
            if not os.path.exists(source_image_path):
                print(f"  Warning: Source image not found at {source_image_path}")
                source_image_path = None

            # Save visualization
            vis_path = os.path.join(output_dir, f"{base_name}_comparison.png")
            visualize_comparison(svg_points, gt_points,
                               title=f"{base_name} - SketchVLM vs GT (Avg Dist: {avg_min_dist:.2f}, MSE Dist: {mse_min_dist:.2f})",
                               save_path=vis_path,
                               background_image_path=source_image_path)

            # Store results
            results.append({
                'svg_file': base_name,
                'gt_identifier': gt_identifier,
                'avg_min_distance': avg_min_dist,
                'mse_min_distance': mse_min_dist,
                'n_points': n_points
            })

            print()

        except Exception as e:
            print(f"Error processing {base_name}: {e}")
            print()
            continue

    # Save summary metrics to CSV
    if results:
        csv_path = os.path.join(output_dir, "metrics_summary.csv")
        with open(csv_path, 'w', newline='') as f:
            writer = csv.DictWriter(f, fieldnames=['svg_file', 'gt_identifier', 'avg_min_distance', 'mse_min_distance', 'n_points'])
            writer.writeheader()
            writer.writerows(results)
        print(f"Saved summary metrics to {csv_path}")

        # Print summary statistics
        avg_distances = [r['avg_min_distance'] for r in results]
        mse_distances = [r['mse_min_distance'] for r in results]
        print(f"\nSummary Statistics ({len(results)} files):")
        print(f"  Average min distance: {np.mean(avg_distances):.2f} ± {np.std(avg_distances):.2f} pixels")
        print(f"  MSE min distance: {np.mean(mse_distances):.2f} ± {np.std(mse_distances):.2f} pixels²")

    return results

def process_combined_batches(model_config, batch_dirs, output_base_dir, n_points=100):
    """
    Process multiple batch directories for a model and combine results.

    Parameters:
    -----------
    model_config : dict
        Configuration with 'name' and 'dirs' keys
    batch_dirs : list
        List of base batch directories to search
    output_base_dir : str
        Base directory for output
    n_points : int
        Number of points to sample

    Returns:
    --------
    list of dict
        Combined results from all batches
    """
    import glob
    import shutil

    model_name = model_config['name']
    dir_patterns = model_config['dirs']

    print(f"\n{'='*80}")
    print(f"Processing model: {model_name}")
    print(f"{'='*80}")

    # Create a temporary directory to hold all SVG files for this model
    temp_dir = os.path.join("/tmp", f"combined_{model_name}")
    os.makedirs(temp_dir, exist_ok=True)

    # Collect all SVG files from all batches
    svg_count = 0
    batch_index = 0
    for batch_dir in batch_dirs:
        for pattern in dir_patterns:
            full_path = os.path.join(batch_dir, pattern)
            if os.path.exists(full_path):
                svg_files = glob.glob(os.path.join(full_path, "*.svg"))
                print(f"Found {len(svg_files)} SVG files in {full_path}")

                # Copy files to temp directory with unique names to avoid collisions
                for svg_file in svg_files:
                    base_name = os.path.basename(svg_file)
                    name_without_ext = os.path.splitext(base_name)[0]
                    json_file = svg_file.replace('.svg', '.json')

                    # Add batch prefix to avoid filename collisions
                    unique_svg_name = f"batch{batch_index}_{base_name}"
                    unique_json_name = f"batch{batch_index}_{name_without_ext}.json"

                    shutil.copy(svg_file, os.path.join(temp_dir, unique_svg_name))
                    if os.path.exists(json_file):
                        shutil.copy(json_file, os.path.join(temp_dir, unique_json_name))
                    svg_count += 1

                batch_index += 1

    print(f"Total SVG files collected: {svg_count}")

    # Create output directory for this model
    output_dir = os.path.join(output_base_dir, model_name)

    # Process all collected files
    results = process_all_svg_files(temp_dir, output_dir, n_points=n_points)

    # Cleanup temp directory
    shutil.rmtree(temp_dir)

    return results

# Usage
if __name__ == "__main__":
    import numpy as np
    import os
    import argparse

    parser = argparse.ArgumentParser(description='Compare SVG ball trajectories with ground truth')
    parser.add_argument('--n_points', type=int, default=100,
                       help='Number of points to sample from each trajectory')

    args = parser.parse_args()

    # Define batch directories
    batch_dirs = [
        '/Users/log/Github/sketchvlm/results/mix_eval/ball_paths/batch1',
        '/Users/log/Github/sketchvlm/results/mix_eval/ball_paths/batch2'
    ]

    # Define models to process
    models = [
        {
            'name': 'gpt5_low',
            'dirs': ['gpt5_low_ball_paths', 'gpt5_low_ball_paths_batch2']
        },
        {
            'name': 'gpt5_med',
            'dirs': ['gpt5_med_ball_paths', 'gpt5_med_ball_paths_batch2']
        },
        {
            'name': 'gemini_25_flash',
            'dirs': ['gemini_25_flash_ball_paths', 'gemini_25_flash_ball_paths_batch2']
        },
        {
            'name': 'gemini_25_pro',
            'dirs': ['gemini_25_pro_ball_paths', 'gemini_25_pro_ball_paths_batch2']
        }
    ]

    # Create output base directory
    script_dir = os.path.dirname(os.path.abspath(__file__))
    output_base_dir = os.path.join(script_dir, "comparisons")

    print("="*80)
    print("Batch Processing: SVG vs Ground Truth Trajectory Comparison")
    print("="*80)
    print(f"Batch directories:")
    for batch_dir in batch_dirs:
        print(f"  - {batch_dir}")
    print(f"\nModels to process:")
    for model in models:
        print(f"  - {model['name']}")
    print(f"\nOutput base directory: {output_base_dir}")
    print(f"Sampling {args.n_points} points per trajectory")
    print("="*80)

    # Process each model
    all_results = {}
    for model in models:
        results = process_combined_batches(model, batch_dirs, output_base_dir, n_points=args.n_points)
        all_results[model['name']] = results

    # Print overall summary
    print("\n" + "="*80)
    print("OVERALL SUMMARY")
    print("="*80)
    for model_name, results in all_results.items():
        if results:
            avg_distances = [r['avg_min_distance'] for r in results]
            mse_distances = [r['mse_min_distance'] for r in results]
            print(f"\n{model_name} ({len(results)} files):")
            print(f"  Average min distance: {np.mean(avg_distances):.2f} ± {np.std(avg_distances):.2f} pixels")
            print(f"  MSE min distance: {np.mean(mse_distances):.2f} ± {np.std(mse_distances):.2f} pixels²")
        else:
            print(f"\n{model_name}: No results")
    print("="*80)