Skip to content

"Vibe-Trading: Your Personal Trading Agent"

12.3 KB
Raw Blame
"""Three-level memory compression pipeline: Raw -> Daily -> Digest (Tier 2).

Uses TF-IDF sentence scoring to extract key information while maintaining
high retention. Original content is archived before compression.

Feature flag: VT_MEMORY_COMPRESSION (default off).
"""

from __future__ import annotations

import logging
import math
import os
import re
import shutil
import time
from collections import Counter
from pathlib import Path
from typing import Dict, List, Optional, Set, Tuple

logger = logging.getLogger(__name__)

# ─── Compression level constants ────────────────────────────────────────────

LEVEL_RAW = "raw"
LEVEL_DAILY = "daily"
LEVEL_DIGEST = "digest"

# Trigger thresholds (days since last access)
DAILY_THRESHOLD_DAYS = 7
DIGEST_THRESHOLD_DAYS = 30

# Compression targets
DAILY_TOP_K_SENTENCES = 5  # Keep top-5 sentences + first/last
DIGEST_MAX_TOKENS = 50  # Max tokens in digest
DIGEST_TOP_KEYWORDS = 15  # Top keywords for digest bullet list

# Seconds per day
_SECONDS_PER_DAY = 86400

# ─── Tokenization ───────────────────────────────────────────────────────────

_NON_LATIN_RANGES = (
    "\u4e00-\u9fff"  # CJK Unified Ideographs
    "\u3400-\u4dbf"  # CJK Extension A
    "\u0e00-\u0e7f"  # Thai
    "\u0620-\u064a"  # Arabic letters
    "\u05d0-\u05ea"  # Hebrew letters
    "\u0400-\u04ff"  # Cyrillic
)
_TOKEN_RE = re.compile(rf"[a-zA-Z0-9]{{3,}}|[{_NON_LATIN_RANGES}]")

# Sentence boundary regex: split on . ! ? and CJK sentence-end marks
_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?。!?])\s+|\n+")


# ─── Public Functions ────────────────────────────────────────────────────────


def _tokenize_for_tfidf(text: str) -> List[str]:
    """Tokenize text for TF-IDF scoring.

    ASCII words >= 3 chars or individual non-Latin script characters.
    Deliberately a higher minimum than persistent.py's retrieval tokenizer:
    this only ranks sentences for summarization, not retrieval, so dropping
    short tokens from scoring doesn't create the silent-miss failure mode
    that a stricter retrieval tokenizer would.
    """
    return _TOKEN_RE.findall(text.lower())


def _split_sentences(text: str) -> List[str]:
    """Split text into sentences.

    Handles English period/question/exclamation and Chinese period
    boundaries. Filters empty results.
    """
    raw_parts = _SENTENCE_SPLIT_RE.split(text.strip())
    sentences = [s.strip() for s in raw_parts if s.strip()]
    return sentences


def compute_tfidf(documents: List[str]) -> Dict[str, float]:
    """Compute IDF scores across a corpus of documents (sentences).

    Args:
        documents: List of document texts (typically sentences).

    Returns:
        Dict mapping term -> IDF score. IDF = log(N / (1 + df(t))).
    """
    n = len(documents)
    if n == 0:
        return {}

    # Count document frequency for each term
    df: Counter = Counter()
    for doc in documents:
        unique_terms = set(_tokenize_for_tfidf(doc))
        for term in unique_terms:
            df[term] += 1

    # Compute IDF: log(N / (1 + df(term)))
    idf_scores: Dict[str, float] = {}
    for term, freq in df.items():
        idf_scores[term] = math.log(n / (1 + freq))

    return idf_scores


def _score_sentence(sentence: str, idf_scores: Dict[str, float]) -> float:
    """Score a sentence by normalized sum of IDF weights.

    Score = sum(idf(token) for token in sentence) / len(tokens).
    Returns 0.0 for empty sentences.
    """
    tokens = _tokenize_for_tfidf(sentence)
    if not tokens:
        return 0.0
    total = sum(idf_scores.get(t, 0.0) for t in tokens)
    return total / len(tokens)


def extract_key_sentences(
    content: str, idf_scores: Dict[str, float], top_k: int = DAILY_TOP_K_SENTENCES
) -> str:
    """Extract top-k sentences ranked by TF-IDF weight sum.

    Always includes first and last sentence for context framing.
    Returns joined text of selected sentences.
    """
    sentences = _split_sentences(content)
    if len(sentences) <= top_k + 2:
        # Already short enough, return as-is
        return content.strip()

    # Score all sentences
    scored: List[Tuple[int, float, str]] = []
    for idx, sent in enumerate(sentences):
        score = _score_sentence(sent, idf_scores)
        scored.append((idx, score, sent))

    # Always keep first and last
    first_idx = 0
    last_idx = len(sentences) - 1
    kept_indices: Set[int] = {first_idx, last_idx}

    # Pick top-k from middle sentences by score
    middle = [(idx, sc, s) for idx, sc, s in scored if idx not in kept_indices]
    middle.sort(key=lambda x: x[1], reverse=True)

    for idx, _sc, _s in middle[:top_k]:
        kept_indices.add(idx)

    # Reconstruct in original order
    selected = [sentences[i] for i in sorted(kept_indices)]
    return " ".join(selected)


# ─── CompressionPipeline ─────────────────────────────────────────────────────


class CompressionPipeline:
    """Three-level compression pipeline for memory entries."""

    def __init__(self, memory_dir: Path) -> None:
        """Initialize with memory directory for archive storage."""
        self._memory_dir = memory_dir
        self._archive_dir = memory_dir / "archive"

    def should_compress(
        self, compression_level: str, last_accessed: float, now: float = 0.0
    ) -> Optional[str]:
        """Determine if an entry should be compressed and to what level.

        Args:
            compression_level: Current level ("raw", "daily", "digest").
            last_accessed: Epoch timestamp of last access.
            now: Current time (defaults to time.time()).

        Returns:
            Target compression level ("daily" or "digest") or None.
        """
        if now <= 0.0:
            now = time.time()

        days_since_access = (now - last_accessed) / _SECONDS_PER_DAY

        if compression_level == LEVEL_RAW and days_since_access > DAILY_THRESHOLD_DAYS:
            return LEVEL_DAILY
        if compression_level == LEVEL_DAILY and days_since_access > DIGEST_THRESHOLD_DAYS:
            return LEVEL_DIGEST

        # Already digest or not enough time elapsed
        return None

    def compress_to_daily(self, content: str, keywords: tuple = ()) -> str:
        """Compress raw content to daily level (~50% size).

        Strategy: TF-IDF key-sentence extraction.
        - Split into sentences
        - Score each sentence by sum of IDF weights
        - Keep top-5 + first sentence + last sentence
        - Prepend keywords as context header
        """
        sentences = _split_sentences(content)
        if not sentences:
            return content

        # Use sentences as document corpus for IDF calculation
        idf_scores = compute_tfidf(sentences)

        # Extract key sentences
        compressed = extract_key_sentences(content, idf_scores, DAILY_TOP_K_SENTENCES)

        # Prepend keywords as context header if available
        if keywords:
            header = "Keywords: " + ", ".join(keywords)
            compressed = header + "\n\n" + compressed

        return compressed

    def compress_to_digest(self, daily_content: str, keywords: tuple = ()) -> str:
        """Compress daily content to digest level (~10-20% of original).

        Strategy: Extract core concepts and keywords.
        - Extract top-N most important terms
        - Format as bullet-point summary
        """
        tokens = _tokenize_for_tfidf(daily_content)
        if not tokens:
            return daily_content

        # Count term frequencies in the content
        tf: Counter = Counter(tokens)

        # Use sentence-level IDF for importance weighting
        sentences = _split_sentences(daily_content)
        idf_scores = compute_tfidf(sentences) if sentences else {}

        # Score each unique term: tf * idf
        term_scores: List[Tuple[str, float]] = []
        for term, freq in tf.items():
            idf = idf_scores.get(term, 1.0)
            term_scores.append((term, freq * idf))

        # Sort by score descending, take top keywords
        term_scores.sort(key=lambda x: x[1], reverse=True)
        top_terms = [t for t, _s in term_scores[:DIGEST_TOP_KEYWORDS]]

        # Build bullet-point summary
        lines: List[str] = []
        if keywords:
            lines.append("Context: " + ", ".join(keywords))
        lines.append("Key concepts:")
        for term in top_terms:
            lines.append(f"  - {term}")

        return "\n".join(lines)

    def archive_original(self, entry_path: Path) -> Optional[Path]:
        """Backup original file to archive/ before compression.

        Returns archive path or None on failure. Creates archive/ dir
        on demand. Uses atomic copy (write tmp + rename) for safety.
        """
        if not entry_path.exists():
            logger.warning("Cannot archive non-existent file: %s", entry_path)
            return None

        try:
            self._archive_dir.mkdir(parents=True, exist_ok=True)
            archive_path = self._archive_dir / entry_path.name
            tmp_path = archive_path.with_suffix(archive_path.suffix + ".tmp")

            # Atomic copy: write to tmp then rename
            shutil.copy2(str(entry_path), str(tmp_path))
            os.replace(str(tmp_path), str(archive_path))

            logger.debug("Archived %s -> %s", entry_path.name, archive_path)
            return archive_path
        except OSError as exc:
            logger.error("Archive failed for %s: %s", entry_path, exc)
            # Clean up tmp file if it exists
            tmp_path = (self._archive_dir / entry_path.name).with_suffix(
                entry_path.suffix + ".tmp"
            )
            if tmp_path.exists():
                try:
                    tmp_path.unlink()
                except OSError:
                    pass
            return None

    def apply_compression(
        self, entry_path: Path, content: str, keywords: tuple, target_level: str
    ) -> Optional[str]:
        """Execute compression: archive original, compute compressed content.

        Args:
            entry_path: Path to the memory .md file.
            content: Current body content.
            keywords: Entry keywords for context.
            target_level: "daily" or "digest".

        Returns:
            Compressed content string, or None on failure.
        """
        # Archive the original before modifying
        archive_result = self.archive_original(entry_path)
        if archive_result is None and entry_path.exists():
            # Archive failed but file exists - abort to avoid data loss
            logger.error(
                "Compression aborted: archive failed for %s", entry_path
            )
            return None

        try:
            if target_level == LEVEL_DAILY:
                compressed = self.compress_to_daily(content, keywords)
            elif target_level == LEVEL_DIGEST:
                compressed = self.compress_to_digest(content, keywords)
            else:
                logger.error("Unknown compression target: %s", target_level)
                return None

            retention = self.estimate_retention(content, compressed)
            logger.info(
                "Compressed %s to %s (retention=%.2f)",
                entry_path.name,
                target_level,
                retention,
            )
            return compressed
        except Exception as exc:
            logger.error("Compression failed for %s: %s", entry_path, exc)
            return None

    def estimate_retention(self, original: str, compressed: str) -> float:
        """Estimate information retention as Jaccard token overlap (0.0-1.0).

        Computes |intersection| / |union| of token sets between original
        and compressed text.
        """
        orig_tokens = set(_tokenize_for_tfidf(original))
        comp_tokens = set(_tokenize_for_tfidf(compressed))

        if not orig_tokens and not comp_tokens:
            return 1.0
        if not orig_tokens or not comp_tokens:
            return 0.0

        intersection = orig_tokens & comp_tokens
        union = orig_tokens | comp_tokens
        return len(intersection) / len(union)