"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
"""Hierarchical directory routing for memory entries (Tier 2).
Organizes memory files into category subdirectories based on memory_type,
enabling O(category_size) scoped searches instead of O(n) flat scans.
Feature flag: VT_MEMORY_HIERARCHY (default off).
"""
from __future__ import annotations
import logging
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, List, Optional, Set
logger = logging.getLogger(__name__)
# Category directories matching MEMORY_TYPES in persistent.py
CATEGORIES = ("user", "feedback", "project", "reference")
# Files/dirs to skip when scanning
_SKIP_NAMES = frozenset({"MEMORY.md", ".hierarchy.yaml", ".lock", "archive", "gc.log"})
@dataclass
class CategorySummary:
"""Summary metadata for a single category directory."""
count: int = 0
keywords: List[str] = field(default_factory=list)
class MemoryHierarchy:
"""Manages hierarchical directory structure for memory entries.
Provides O(category_size) routed access to memory files organized
by memory_type, while maintaining backward compatibility with
flat-stored entries in the base directory.
"""
def __init__(self, base_dir: Path) -> None:
"""Initialize with the memory base directory.
Args:
base_dir: Root directory for memory storage
(e.g. ~/.vibe-trading/memory/).
"""
self._base_dir = base_dir
self._index_path = base_dir / ".hierarchy.yaml"
@property
def base_dir(self) -> Path:
"""Return the configured base directory."""
return self._base_dir
def _ensure_category_dir(self, category: str) -> Path:
"""Create category subdirectory if it does not exist.
Args:
category: One of CATEGORIES or any string.
Returns:
Path to the category subdirectory.
"""
cat_dir = self._base_dir / category
cat_dir.mkdir(parents=True, exist_ok=True)
return cat_dir
def route_entry(self, memory_type: str, filename: str) -> Path:
"""Determine storage path: base_dir/{memory_type}/{filename}.
Creates category directory on demand.
Falls back to base_dir for unknown types.
Args:
memory_type: Category identifier (e.g. "user", "project").
filename: The .md filename for the memory entry.
Returns:
Full path where the entry should be stored.
"""
if memory_type in CATEGORIES:
cat_dir = self._ensure_category_dir(memory_type)
return cat_dir / filename
# Unknown type falls back to base directory
logger.warning(
"Unknown memory_type '%s', routing to base dir", memory_type
)
return self._base_dir / filename
def recover_extensionless_entries(self) -> List[Path]:
"""Rename category entries that were written without the ``.md`` suffix.
``route_entry`` documents its second argument as "the .md filename",
but a caller passed a bare slug, so entries landed as
``<category>/<slug>`` with no suffix. Every scan here filters on
``suffix == ".md"``, which means those entries are not merely
mis-named — they are invisible to ``list_entries`` and ``find``, and
stay invisible after the write path is corrected, because nothing
goes back for them.
Only a suffix-less file whose content opens with a frontmatter fence
is touched, so an unrelated file dropped in a category directory is
left alone. An existing ``<slug>.md`` wins; the orphan is left in
place rather than silently overwriting the newer entry.
Returns:
Paths of the entries after renaming, in the order recovered.
"""
recovered: List[Path] = []
for category in CATEGORIES:
cat_dir = self._base_dir / category
if not cat_dir.is_dir():
continue
for item in sorted(cat_dir.iterdir()):
if item.name in _SKIP_NAMES or not item.is_file():
continue
if item.suffix:
continue
target = item.with_suffix(".md")
if target.exists():
logger.warning(
"Extension-less memory entry %s shadowed by %s; leaving "
"it in place rather than overwriting the newer entry",
item,
target,
)
continue
try:
head = item.read_text(encoding="utf-8")[:3]
except OSError:
continue
if head.strip() != "---":
continue
try:
item.rename(target)
except OSError:
logger.warning("Could not recover memory entry %s", item)
continue
logger.info("Recovered memory entry %s -> %s", item.name, target.name)
recovered.append(target)
return recovered
def scan_all(self) -> List[Path]:
"""Scan base dir + all category subdirs for *.md files.
Skips entries listed in _SKIP_NAMES and the archive/ directory.
Includes flat-stored entries in base dir for backward compatibility.
Entries orphaned by the missing-suffix write bug are recovered first,
so they become visible without the user having to run anything.
Returns:
Sorted list of .md file paths.
"""
self.recover_extensionless_entries()
results: List[Path] = []
# Scan base directory (flat legacy entries)
if self._base_dir.is_dir():
for item in self._base_dir.iterdir():
if item.name in _SKIP_NAMES:
continue
if item.is_file() and item.suffix == ".md":
results.append(item)
# Scan each known category subdirectory
for category in CATEGORIES:
cat_dir = self._base_dir / category
if cat_dir.is_dir():
for item in cat_dir.iterdir():
if item.is_file() and item.suffix == ".md":
results.append(item)
results.sort(key=lambda p: p.name)
return results
def scan_category(self, category: str) -> List[Path]:
"""Scan only one category subdir for *.md files.
Args:
category: The category name to scan.
Returns:
Sorted list of .md file paths in that category directory.
"""
cat_dir = self._base_dir / category
results: List[Path] = []
if not cat_dir.is_dir():
return results
for item in cat_dir.iterdir():
if item.is_file() and item.suffix == ".md":
results.append(item)
results.sort(key=lambda p: p.name)
return results
def rebuild_index(self, entries: list) -> None:
"""Write .hierarchy.yaml with per-category summary.
Format:
categories:
<name>:
count: <int>
keywords: [kw1, kw2, ...]
rebuilt_at: <ISO-8601>
Uses manual YAML formatting (no PyYAML dependency).
Args:
entries: List of dicts with at least 'memory_type' key and
optionally 'keywords' (list of strings).
"""
# Aggregate per-category statistics
cat_data: Dict[str, CategorySummary] = {
cat: CategorySummary() for cat in CATEGORIES
}
for entry in entries:
mtype = entry.get("memory_type", "")
if mtype not in cat_data:
continue
cat_data[mtype].count += 1
# Collect keywords (deduplicate later)
keywords = entry.get("keywords", [])
if isinstance(keywords, list):
cat_data[mtype].keywords.extend(keywords)
# Deduplicate and limit keywords per category
max_keywords = 10 # keep index compact
for summary in cat_data.values():
seen: Set[str] = set()
unique: List[str] = []
for kw in summary.keywords:
kw_lower = kw.lower().strip()
if kw_lower and kw_lower not in seen:
seen.add(kw_lower)
unique.append(kw_lower)
summary.keywords = unique[:max_keywords]
# Generate ISO-8601 timestamp
rebuilt_at = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime())
# Write YAML manually
lines: List[str] = [
"# Auto-generated memory hierarchy index",
f'rebuilt_at: "{rebuilt_at}"',
"categories:",
]
for cat in CATEGORIES:
summary = cat_data[cat]
kw_list = ", ".join(summary.keywords)
lines.append(f" {cat}:")
lines.append(f" count: {summary.count}")
lines.append(f" keywords: [{kw_list}]")
self._base_dir.mkdir(parents=True, exist_ok=True)
self._index_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
logger.debug("Rebuilt hierarchy index at %s", self._index_path)
def _parse_index_keywords(self) -> Dict[str, List[str]]:
"""Parse .hierarchy.yaml to extract per-category keywords.
Returns:
Dict mapping category name to list of keywords.
Empty dict if index file does not exist or is malformed.
"""
if not self._index_path.is_file():
return {}
result: Dict[str, List[str]] = {}
current_cat: Optional[str] = None
try:
text = self._index_path.read_text(encoding="utf-8")
except OSError:
return {}
for line in text.splitlines():
stripped = line.strip()
# Detect category header (e.g. " user:")
if (
stripped.endswith(":")
and not stripped.startswith("#")
and not stripped.startswith("categories")
and not stripped.startswith("rebuilt_at")
):
cat_name = stripped.rstrip(":")
if cat_name in CATEGORIES:
current_cat = cat_name
if current_cat not in result:
result[current_cat] = []
else:
current_cat = None
# Detect keywords line (e.g. " keywords: [a, b, c]")
elif stripped.startswith("keywords:") and current_cat:
# Extract content between brackets
bracket_start = stripped.find("[")
bracket_end = stripped.find("]")
if bracket_start != -1 and bracket_end != -1:
inner = stripped[bracket_start + 1 : bracket_end]
keywords = [
k.strip() for k in inner.split(",") if k.strip()
]
result[current_cat] = keywords
return result
def prune_search_scope(
self, query_tokens: Set[str], category_filter: str = ""
) -> List[Path]:
"""Narrow search scope based on category filter or keyword overlap.
If category_filter is set, only scan that category.
Otherwise scan all categories (maintains compatibility).
When no category_filter is provided and an index exists, categories
are ordered by keyword overlap score (most relevant first), but all
files are still returned for completeness.
Args:
query_tokens: Set of lowercase query terms for relevance scoring.
category_filter: If non-empty, restrict scan to this category.
Returns:
List of .md file paths in priority order.
"""
# Direct category filter — fast path
if category_filter:
if category_filter in CATEGORIES:
return self.scan_category(category_filter)
# Unknown filter: fall back to full scan
logger.warning(
"Unknown category_filter '%s', falling back to scan_all",
category_filter,
)
return self.scan_all()
# No filter: use keyword overlap to order categories
index_keywords = self._parse_index_keywords()
if not index_keywords:
# No index available — full scan
return self.scan_all()
# Score each category by keyword overlap with query tokens
scored: List[tuple] = []
for cat in CATEGORIES:
cat_kws = set(index_keywords.get(cat, []))
overlap = len(query_tokens & cat_kws)
scored.append((overlap, cat))
# Sort descending by overlap score
scored.sort(key=lambda x: x[0], reverse=True)
# Collect files: prioritized categories first, then base dir
results: List[Path] = []
seen: Set[Path] = set()
for _score, cat in scored:
for p in self.scan_category(cat):
if p not in seen:
results.append(p)
seen.add(p)
# Include flat base-dir entries for backward compatibility
if self._base_dir.is_dir():
for item in self._base_dir.iterdir():
if item.name in _SKIP_NAMES:
continue
if item.is_file() and item.suffix == ".md" and item not in seen:
results.append(item)
seen.add(item)
return results
def migrate_flat_entry(
self, file_path: Path, memory_type: str
) -> Optional[Path]:
"""Move a flat-stored entry to its category subdir.
Only moves if the file currently resides in base_dir (not already
in a subdirectory). Preserves file content unchanged.
Args:
file_path: Current path of the memory .md file.
memory_type: Target category for the file.
Returns:
New path after migration, or None if migration was skipped
(file not in base dir, target same as current, or error).
"""
# Validate source exists and is in base directory
if not file_path.is_file():
logger.warning("Cannot migrate non-existent file: %s", file_path)
return None
if file_path.parent != self._base_dir:
logger.debug(
"File %s not in base dir, skip migration", file_path.name
)
return None
if memory_type not in CATEGORIES:
logger.warning(
"Cannot migrate to unknown category '%s'", memory_type
)
return None
# Determine destination
dest_dir = self._ensure_category_dir(memory_type)
dest_path = dest_dir / file_path.name
if dest_path.exists():
logger.warning(
"Destination already exists, skip migration: %s", dest_path
)
return None
try:
file_path.rename(dest_path)
logger.info(
"Migrated %s -> %s/%s",
file_path.name,
memory_type,
file_path.name,
)
return dest_path
except OSError as exc:
logger.error("Migration failed for %s: %s", file_path.name, exc)
return None