Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Parallelize headroom learn session scanning with --workers flag

chopratejas6cc8fd04b6ba
diff --git a/headroom/cli/learn.py b/headroom/cli/learn.py
index 7f4be73f4..b0c6f4bf2 100644
--- a/headroom/cli/learn.py
+++ b/headroom/cli/learn.py
@@ -90,12 +90,21 @@ Use 'auto' (default) to scan all detected agents."""
help="LLM model for analysis (e.g., claude-sonnet-4-6, gpt-4o, gemini/gemini-2.0-flash). "
"Auto-detected from API keys if not specified.",
)
+ "--workers",
+ "-j",
+ type=int,
+ default=None,
+ help="Parallel workers for session scanning. "
+ "Default: auto (min of CPU count, 8). Use 1 for serial.",
+)
def learn(
project: Path | None,
analyze_all: bool,
apply: bool,
agent: str,
model: str | None,
+ workers: int | None,
) -> None:
"""Learn from past tool call failures to prevent future ones.
@@ -115,9 +124,13 @@ def learn(
headroom learn --all # Analyze all projects
headroom learn --agent codex --all # Analyze all Codex sessions
"""
+ import os
+
from ..learn.analyzer import SessionAnalyzer, _detect_default_model
from ..learn.registry import auto_detect_plugins, get_plugin
+ max_workers = workers if workers is not None else min(os.cpu_count() or 4, 8)
+
# Resolve model early to fail fast with a clear message
try:
resolved_model = model or _detect_default_model()
@@ -185,7 +198,7 @@ def learn(
click.echo(f"Path: {proj.project_path}")
click.echo(f"{'=' * 60}")
- sessions = plugin.scan_project(proj)
+ sessions = plugin.scan_project(proj, max_workers=max_workers)
if not sessions:
click.echo(" No conversation data found.")
continue
diff --git a/headroom/learn/base.py b/headroom/learn/base.py
index 99feefefa..2d77ea3b2 100644
--- a/headroom/learn/base.py
+++ b/headroom/learn/base.py
@@ -25,7 +25,7 @@ class ConversationScanner(ABC):
...
@abstractmethod
- def scan_project(self, project: ProjectInfo) -> list[SessionData]:
+ def scan_project(self, project: ProjectInfo, max_workers: int = 1) -> list[SessionData]:
"""Scan all sessions for a project, returning normalized tool calls."""
...
@@ -99,8 +99,14 @@ class LearnPlugin(ABC):
...
@abstractmethod
- def scan_project(self, project: ProjectInfo) -> list[SessionData]:
- """Scan all sessions for a project, returning normalized data."""
+ def scan_project(self, project: ProjectInfo, max_workers: int = 1) -> list[SessionData]:
+ """Scan all sessions for a project, returning normalized data.
+
+ Args:
+ project: The project to scan.
+ max_workers: Number of threads for parallel file scanning.
+ 1 (default) = serial. >1 = concurrent.
+ """
...
# --- Writing ---
diff --git a/headroom/learn/plugins/claude.py b/headroom/learn/plugins/claude.py
index 049b57788..1275a850f 100644
--- a/headroom/learn/plugins/claude.py
+++ b/headroom/learn/plugins/claude.py
@@ -106,16 +106,24 @@ class ClaudeCodePlugin(LearnPlugin, ConversationScanner):
return projects
- def scan_project(self, project: ProjectInfo) -> list[SessionData]:
+ def scan_project(self, project: ProjectInfo, max_workers: int = 1) -> list[SessionData]:
"""Scan all conversation JSONL files for a project."""
- sessions = []
jsonl_files = sorted(project.data_path.glob("*.jsonl"))
+ if not jsonl_files:
+ return []
+
+ if max_workers <= 1 or len(jsonl_files) <= 1:
+ return [s for f in jsonl_files if (s := self._scan_session(f)) and s.tool_calls]
- for jsonl_path in jsonl_files:
- session = self._scan_session(jsonl_path)
- if session and session.tool_calls:
- sessions.append(session)
+ from concurrent.futures import ThreadPoolExecutor, as_completed
+ sessions: list[SessionData] = []
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ futures = {executor.submit(self._scan_session, f): f for f in jsonl_files}
+ for future in as_completed(futures):
+ session = future.result()
+ if session and session.tool_calls:
+ sessions.append(session)
return sessions
def _scan_session(self, jsonl_path: Path) -> SessionData | None:
diff --git a/headroom/learn/plugins/codex.py b/headroom/learn/plugins/codex.py
index 96f507f0b..774fc5fae 100644
--- a/headroom/learn/plugins/codex.py
+++ b/headroom/learn/plugins/codex.py
@@ -91,13 +91,24 @@ class CodexPlugin(LearnPlugin, ConversationScanner):
)
]
- def scan_project(self, project: ProjectInfo) -> list[SessionData]:
+ def scan_project(self, project: ProjectInfo, max_workers: int = 1) -> list[SessionData]:
"""Scan all Codex session JSON files."""
- sessions = []
- for json_path in self._iter_session_files(project.data_path):
- session = self._scan_session(json_path)
- if session and session.tool_calls:
- sessions.append(session)
+ session_files = self._iter_session_files(project.data_path)
+ if not session_files:
+ return []
+
+ if max_workers <= 1 or len(session_files) <= 1:
+ return [s for f in session_files if (s := self._scan_session(f)) and s.tool_calls]
+
+ from concurrent.futures import ThreadPoolExecutor, as_completed
+
+ sessions: list[SessionData] = []
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ futures = {executor.submit(self._scan_session, f): f for f in session_files}
+ for future in as_completed(futures):
+ session = future.result()
+ if session and session.tool_calls:
+ sessions.append(session)
return sessions
def _scan_session(self, json_path: Path) -> SessionData | None:
diff --git a/headroom/learn/plugins/gemini.py b/headroom/learn/plugins/gemini.py
index ead536184..e7e1871ca 100644
--- a/headroom/learn/plugins/gemini.py
+++ b/headroom/learn/plugins/gemini.py
@@ -106,18 +106,26 @@ class GeminiPlugin(LearnPlugin, ConversationScanner):
return projects
- def scan_project(self, project: ProjectInfo) -> list[SessionData]:
+ def scan_project(self, project: ProjectInfo, max_workers: int = 1) -> list[SessionData]:
"""Scan all Gemini session files for a project."""
- sessions = []
session_files = sorted(project.data_path.glob("session-*.json")) + sorted(
project.data_path.glob("session-*.jsonl")
)
+ if not session_files:
+ return []
+
+ if max_workers <= 1 or len(session_files) <= 1:
+ return [s for f in session_files if (s := self._scan_session(f)) and s.tool_calls]
- for session_path in session_files:
- session = self._scan_session(session_path)
- if session and session.tool_calls:
- sessions.append(session)
+ from concurrent.futures import ThreadPoolExecutor, as_completed
+ sessions: list[SessionData] = []
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ futures = {executor.submit(self._scan_session, f): f for f in session_files}
+ for future in as_completed(futures):
+ session = future.result()
+ if session and session.tool_calls:
+ sessions.append(session)
return sessions
def _scan_session(self, session_path: Path) -> SessionData | None: