Skip to content

The agent that turns websites into APIs!

10.4 KB
Raw Blame
"""Real-time file synchronization with watchdog."""

import os
import shutil
import time
from collections.abc import Callable
from datetime import datetime
from pathlib import Path
from threading import Event, Thread
from typing import TypedDict

from watchdog.events import FileSystemEvent, FileSystemEventHandler
from watchdog.observers import Observer


def get_available_directory(base_path: Path, base_name: str) -> Path:
    """
    Find an available directory name, appending timestamp if needed.

    If the base directory exists and is not empty, returns a new directory
    with timestamp suffix. Otherwise returns the base directory.
    """
    target_dir = base_path / base_name

    # If directory doesn't exist or is empty, use it
    if not target_dir.exists() or (target_dir.is_dir() and not any(target_dir.iterdir())):
        return target_dir

    # Otherwise, create a new directory with timestamp
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    folder_name = f"{base_name}_{timestamp}"
    return base_path / folder_name


class PendingSyncEvent(TypedDict):
    time: float
    is_delete: bool


class SyncHandler(FileSystemEventHandler):
    """Handle file system events for syncing."""

    def __init__(
        self,
        source_dir: Path,
        dest_dir: Path,
        on_sync: Callable[[str], None] | None = None,
        on_error: Callable[[str], None] | None = None,
        debounce_ms: int = 500,
    ) -> None:
        self.source_dir = source_dir
        self.dest_dir = dest_dir
        self.on_sync = on_sync
        self.on_error = on_error
        self.debounce_ms = debounce_ms / 1000.0  # Convert to seconds
        self.pending_events: dict[str, PendingSyncEvent] = {}
        self.last_sync_time = 0.0
        self.file_count = 0

    def _is_ignored_file(self, file_path: str) -> bool:
        """Check if a file should be ignored during sync."""
        path = Path(file_path)
        try:
            relative = path.relative_to(self.source_dir)
        except ValueError:
            relative = path
        return _should_skip_path(relative)

    def _is_temporary_file(self, file_path: str) -> bool:
        """Backward-compatible helper for temporary/ignored sync paths."""
        return self._is_ignored_file(file_path)

    def on_created(self, event: FileSystemEvent) -> None:
        """Handle file creation."""
        if not event.is_directory and not self._is_ignored_file(os.fsdecode(event.src_path)):
            self._queue_sync(os.fsdecode(event.src_path))

    def on_modified(self, event: FileSystemEvent) -> None:
        """Handle file modification."""
        if not event.is_directory and not self._is_ignored_file(os.fsdecode(event.src_path)):
            self._queue_sync(os.fsdecode(event.src_path))

    def on_deleted(self, event: FileSystemEvent) -> None:
        """Handle file deletion."""
        if not event.is_directory and not self._is_ignored_file(os.fsdecode(event.src_path)):
            self._queue_sync(os.fsdecode(event.src_path), is_delete=True)

    def _queue_sync(self, file_path: str, is_delete: bool = False) -> None:
        """Queue a file for syncing with debouncing."""
        self.pending_events[file_path] = {
            "time": time.time(),
            "is_delete": is_delete,
        }

    def process_pending(self) -> None:
        """Process pending sync events (debounced)."""
        current_time = time.time()
        to_sync = []

        # Find events that have passed the debounce period
        for file_path, event_data in list(self.pending_events.items()):
            if current_time - event_data["time"] >= self.debounce_ms:
                to_sync.append((file_path, event_data["is_delete"]))
                del self.pending_events[file_path]

        # Sync the files
        for file_path, is_delete in to_sync:
            try:
                self._sync_file(file_path, is_delete)
            except Exception as e:
                if self.on_error:
                    self.on_error(f"Error syncing {Path(file_path).name}: {str(e)}")

    def _sync_file(self, source_path: str, is_delete: bool = False) -> None:
        """Sync a single file from source to destination."""
        source = Path(source_path)
        relative = source.relative_to(self.source_dir)
        dest = self.dest_dir / relative

        if is_delete:
            # Delete from destination
            if dest.exists():
                dest.unlink()
                self.file_count -= 1
        else:
            # Check if source file exists before attempting to copy
            if not source.exists():
                return

            # Copy to destination
            dest.parent.mkdir(parents=True, exist_ok=True)
            try:
                shutil.copy2(source, dest)
                if dest.exists():
                    self.file_count += 1
            except FileNotFoundError:
                # File was deleted between existence check and copy, skip silently
                return

        # Update last sync time
        self.last_sync_time = time.time()

        # Notify of sync
        if self.on_sync:
            action = "Deleted" if is_delete else "Synced"
            self.on_sync(f"{action} {source.name}")


class FileSyncWatcher:
    """Watch a directory and sync files in real-time."""

    def __init__(
        self,
        source_dir: Path,
        dest_dir: Path,
        on_sync: Callable[[str], None] | None = None,
        on_error: Callable[[str], None] | None = None,
        debounce_ms: int = 500,
    ) -> None:
        self.source_dir = source_dir
        self.dest_dir = dest_dir
        self.debounce_ms = debounce_ms

        # Create handler
        self.handler = SyncHandler(
            source_dir=source_dir,
            dest_dir=dest_dir,
            on_sync=on_sync,
            on_error=on_error,
            debounce_ms=debounce_ms,
        )

        # Create observer
        self.observer = Observer()
        self.observer.schedule(self.handler, str(source_dir), recursive=True)

        # Thread control
        self.stop_event = Event()
        self.process_thread: Thread | None = None

    def start(self) -> None:
        """Start watching and syncing."""
        # Ensure destination exists
        self.dest_dir.mkdir(parents=True, exist_ok=True)

        # Start observer
        self.observer.start()

        # Start processing thread for debouncing
        self.process_thread = Thread(target=self._process_loop, daemon=True)
        self.process_thread.start()

    def stop(self) -> None:
        """Stop watching and syncing."""
        self.stop_event.set()

        # Process any remaining pending events before stopping
        # Give it a moment to process debounced events
        time.sleep(self.debounce_ms / 1000.0 + 0.1)
        self.handler.process_pending()

        self.observer.stop()
        self.observer.join(timeout=2)

        if self.process_thread:
            self.process_thread.join(timeout=2)

        # Perform final sync of all existing files to ensure nothing is missed
        self._final_sync()

    def flush(self) -> None:
        """Flush pending events and perform a full sync without stopping the watcher."""
        self.handler.process_pending()
        self._final_sync()

    def _process_loop(self) -> None:
        """Background loop to process pending sync events."""
        while not self.stop_event.is_set():
            self.handler.process_pending()
            time.sleep(0.1)  # Check every 100ms

    def _final_sync(self) -> None:
        """Perform a final sync of all existing files in source directory."""
        if not self.source_dir.exists():
            return

        for item in self.source_dir.rglob("*"):
            if not item.is_file():
                continue
            relative = item.relative_to(self.source_dir)
            if _should_skip_path(relative):
                continue
            dest = self.dest_dir / relative

            # Only sync if destination doesn't exist or is older
            try:
                if not dest.exists() or item.stat().st_mtime > dest.stat().st_mtime:
                    dest.parent.mkdir(parents=True, exist_ok=True)
                    shutil.copy2(item, dest)
                    if self.handler.on_sync:
                        self.handler.on_sync(f"Synced {item.name}")
            except (FileNotFoundError, OSError):
                # File was deleted or inaccessible, skip silently
                pass

    def get_status(self) -> dict:
        """Get current sync status."""
        if not self.observer.is_alive():
            return {
                "active": False,
                "error": "Observer stopped",
            }

        seconds_since_sync = time.time() - self.handler.last_sync_time if self.handler.last_sync_time > 0 else 0

        return {
            "active": True,
            "last_sync": f"{int(seconds_since_sync)}s ago" if self.handler.last_sync_time > 0 else "never",
            "file_count": self.handler.file_count,
        }


def _should_skip_path(relative_path: Path) -> bool:
    """Check if a relative path should be skipped during sync.

    Checks for ignored directories (node_modules, __pycache__) and
    temporary file patterns (.tmp, .swp, ~-prefixed).
    """
    name = relative_path.name

    # Check for ignored directories
    skip_dirs = {"node_modules", "__pycache__"}
    if skip_dirs & set(relative_path.parts):
        return True

    # Check for temporary file patterns
    if name.endswith(".tmp") or ".tmp." in name:
        return True
    if name.startswith(".") and name.endswith(".swp"):
        return True
    if name.startswith("~"):
        return True

    return False


def sync_directory_once(source_dir: Path, dest_dir: Path) -> Path:
    """
    Perform a one-time sync of a directory.

    If dest_dir already exists and is not empty, finds an available
    directory with _iter{i} suffix instead of overwriting.
    """
    # Extract base path and name from dest_dir
    base_path = dest_dir.parent
    base_name = dest_dir.name

    # Get available directory (won't overwrite existing non-empty dirs)
    final_dest_dir = get_available_directory(base_path, base_name)
    final_dest_dir.mkdir(parents=True, exist_ok=True)

    for item in source_dir.rglob("*"):
        if item.is_file():
            relative = item.relative_to(source_dir)
            if _should_skip_path(relative):
                continue
            dest = final_dest_dir / relative
            dest.parent.mkdir(parents=True, exist_ok=True)
            shutil.copy2(item, dest)

    return final_dest_dir