Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

chore: add nosec B324 annotations to non-cryptographic MD5 usages and update temporary database path to use system temp directory

rrubayet3212ae71fe44df4
diff --git a/benchmarks/compression_benchmark.py b/benchmarks/compression_benchmark.py
index 7f7fa7c85..a3b17edf7 100644
--- a/benchmarks/compression_benchmark.py
+++ b/benchmarks/compression_benchmark.py
@@ -159,7 +159,7 @@ def generate_log_data(
entry = random.choice(log_templates).copy()
entry["timestamp"] = f"2024-01-15T{10 + (i // 60):02d}:{i % 60:02d}:00Z"
- entry["request_id"] = f"req-{hashlib.md5(str(i).encode()).hexdigest()[:8]}"
+ entry["request_id"] = f"req-{hashlib.md5(str(i).encode()).hexdigest()[:8]}" # nosec B324
logs.append(entry)
# Questions designed to test different approaches
diff --git a/benchmarks/headroom_adversarial_benchmark.py b/benchmarks/headroom_adversarial_benchmark.py
index 89ea9fb2c..fabd7f1ce 100644
--- a/benchmarks/headroom_adversarial_benchmark.py
+++ b/benchmarks/headroom_adversarial_benchmark.py
@@ -174,7 +174,7 @@ def generate_code_diff(num_files: int = 15, changes_per_file: int = 20) -> dict:
filename = f"src/module_{file_idx}/handler.{ext}"
diff_output += f"diff --git a/{filename} b/{filename}\n"
- diff_output += f"index {hashlib.md5(f'{file_idx}a'.encode()).hexdigest()[:7]}..{hashlib.md5(f'{file_idx}b'.encode()).hexdigest()[:7]} 100644\n"
+ diff_output += f"index {hashlib.md5(f'{file_idx}a'.encode()).hexdigest()[:7]}..{hashlib.md5(f'{file_idx}b'.encode()).hexdigest()[:7]} 100644\n" # nosec B324
diff_output += f"--- a/{filename}\n"
diff_output += f"+++ b/{filename}\n"
@@ -227,7 +227,7 @@ def generate_encrypted_data(size_kb: int = 20) -> dict:
return {
"tool": "encrypted_blob",
"result": {
- "blob_id": f"enc_{hashlib.md5(encoded[:100].encode()).hexdigest()[:16]}",
+ "blob_id": f"enc_{hashlib.md5(encoded[:100].encode()).hexdigest()[:16]}", # nosec B324
"encryption": "AES-256-GCM",
"content": encoded,
"size_bytes": len(random_bytes),
@@ -246,7 +246,7 @@ def generate_tiny_dataset(num_items: int = 5) -> dict:
"id": i + 1,
"name": f"Item {chr(65 + i)}",
"value": random.randint(100, 999),
- "note": f"Unique note for item {i + 1}: {hashlib.md5(str(i).encode()).hexdigest()[:20]}",
+ "note": f"Unique note for item {i + 1}: {hashlib.md5(str(i).encode()).hexdigest()[:20]}", # nosec B324
}
)
@@ -422,7 +422,7 @@ def run_scenario(
# Add tool results
for tool_output in scenario.tools:
- tool_call_id = f"call_{hashlib.md5(tool_output['tool'].encode()).hexdigest()[:8]}"
+ tool_call_id = f"call_{hashlib.md5(tool_output['tool'].encode()).hexdigest()[:8]}" # nosec B324
messages.append(
{
"role": "assistant",
diff --git a/benchmarks/headroom_worst_case_benchmark.py b/benchmarks/headroom_worst_case_benchmark.py
index c441787e1..a28480f28 100644
--- a/benchmarks/headroom_worst_case_benchmark.py
+++ b/benchmarks/headroom_worst_case_benchmark.py
@@ -100,14 +100,14 @@ def generate_unique_support_tickets(num_tickets: int = 50) -> dict:
days=random.randint(2, 14),
from_plan=random.choice(["Enterprise", "Pro"]),
to_plan=random.choice(["Starter", "Team"]),
- user_id=f"usr_{hashlib.md5(str(i).encode()).hexdigest()[:8]}",
+ user_id=f"usr_{hashlib.md5(str(i).encode()).hexdigest()[:8]}", # nosec B324
)
tickets.append(
{
"ticket_id": f"TKT-{20000 + i}",
"customer": {
- "id": f"cust_{hashlib.md5(f'customer{i}'.encode()).hexdigest()[:8]}",
+ "id": f"cust_{hashlib.md5(f'customer{i}'.encode()).hexdigest()[:8]}", # nosec B324
"name": f"Customer {i + 1}",
"company": f"Company {chr(65 + (i % 26))}{i // 26 + 1} Inc.",
"plan": random.choice(products),
@@ -159,7 +159,7 @@ def generate_unique_error_traces(num_traces: int = 30) -> dict:
traces.append(
{
- "error_id": f"err_{hashlib.md5(str(i).encode()).hexdigest()[:12]}",
+ "error_id": f"err_{hashlib.md5(str(i).encode()).hexdigest()[:12]}", # nosec B324
"timestamp": f"2024-01-17T{10 + (i % 12):02d}:{(i * 7) % 60:02d}:00Z",
"service": random.choice(["api", "worker", "scheduler", "gateway"]),
"environment": "production",
@@ -169,7 +169,7 @@ def generate_unique_error_traces(num_traces: int = 30) -> dict:
"stack_trace": trace["stack"],
"context": {
"user_id": f"user_{random.randint(10000, 99999)}",
- "request_id": hashlib.md5(f"req{i}".encode()).hexdigest()[:16],
+ "request_id": hashlib.md5(f"req{i}".encode()).hexdigest()[:16], # nosec B324
"endpoint": trace.get("endpoint", "/api/unknown"),
},
"occurrence_count": random.randint(1, 5), # Low count - each is unique
@@ -630,7 +630,7 @@ def run_scenario(
# Add tool results with proper format
for tool_output in scenario.tools:
- tool_call_id = f"call_{hashlib.md5(tool_output['tool'].encode()).hexdigest()[:8]}"
+ tool_call_id = f"call_{hashlib.md5(tool_output['tool'].encode()).hexdigest()[:8]}" # nosec B324
messages.append(
{
"role": "assistant",
diff --git a/benchmarks/real_world_agent_benchmark.py b/benchmarks/real_world_agent_benchmark.py
index 7b4c8882e..cb64c65d4 100644
--- a/benchmarks/real_world_agent_benchmark.py
+++ b/benchmarks/real_world_agent_benchmark.py
@@ -386,7 +386,7 @@ def generate_log_search(query: str, num_entries: int = 300) -> dict:
"level": level,
"service": service,
"message": message,
- "trace_id": hashlib.md5(f"{i}".encode()).hexdigest()[:16],
+ "trace_id": hashlib.md5(f"{i}".encode()).hexdigest()[:16], # nosec B324
"metadata": {
"host": f"pod-{service}-{random.randint(1, 5)}",
"region": random.choice(["us-east-1", "us-west-2", "eu-west-1"]),
@@ -518,7 +518,7 @@ def run_agent_scenario(
# Add tool results with proper OpenAI format
for tool_output in scenario.tools:
- tool_call_id = f"call_{hashlib.md5(tool_output['tool'].encode()).hexdigest()[:8]}"
+ tool_call_id = f"call_{hashlib.md5(tool_output['tool'].encode()).hexdigest()[:8]}" # nosec B324
# Assistant message with tool_calls (required by OpenAI)
messages.append(
{
diff --git a/headroom/cache/base.py b/headroom/cache/base.py
index 14239f3f4..096eb72ef 100644
--- a/headroom/cache/base.py
+++ b/headroom/cache/base.py
@@ -320,7 +320,7 @@ class BaseCacheOptimizer(ABC):
"""Compute a short hash of content."""
import hashlib
- return hashlib.md5(content.encode()).hexdigest()[:12]
+ return hashlib.md5(content.encode()).hexdigest()[:12] # nosec B324
def _extract_system_content(self, messages: list[dict[str, Any]]) -> str:
"""Extract content from system messages."""
diff --git a/headroom/cache/compression_cache.py b/headroom/cache/compression_cache.py
index 247026bb4..a8cfc5f53 100644
--- a/headroom/cache/compression_cache.py
+++ b/headroom/cache/compression_cache.py
@@ -135,7 +135,7 @@ class CompressionCache:
raw = json.dumps(content, sort_keys=True, ensure_ascii=False)
else:
raw = content
- return hashlib.md5(raw.encode("utf-8")).hexdigest()[:16]
+ return hashlib.md5(raw.encode("utf-8")).hexdigest()[:16] # nosec B324
def compute_frozen_count(self, messages: list[dict]) -> int:
"""Count consecutive stable messages from the start.
diff --git a/headroom/cache/compression_store.py b/headroom/cache/compression_store.py
index 0b6389b4a..f6ad6f54e 100644
--- a/headroom/cache/compression_store.py
+++ b/headroom/cache/compression_store.py
@@ -206,7 +206,7 @@ class CompressionStore:
# collision resistance. Birthday paradox: 50% collision at sqrt(2^n) entries.
# - 64 bits: ~4 billion entries for 50% collision
# - 96 bits: ~280 trillion entries for 50% collision
- hash_key = hashlib.md5(original.encode()).hexdigest()[:24]
+ hash_key = hashlib.md5(original.encode()).hexdigest()[:24] # nosec B324
entry = CompressionEntry(
hash=hash_key,
diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.py
index 77561ca23..56453bb3d 100644
--- a/headroom/cache/prefix_tracker.py
+++ b/headroom/cache/prefix_tracker.py
@@ -305,7 +305,7 @@ class SessionTrackerStore:
break
key = f"{model}:{system_content}"
- return hashlib.md5(key.encode()).hexdigest()[:16]
+ return hashlib.md5(key.encode()).hexdigest()[:16] # nosec B324
def _maybe_cleanup(self) -> None:
"""Remove expired trackers periodically."""
diff --git a/headroom/evals/memory/runner_v2.py b/headroom/evals/memory/runner_v2.py
index a25a72b80..e42ff56cb 100644
--- a/headroom/evals/memory/runner_v2.py
+++ b/headroom/evals/memory/runner_v2.py
@@ -17,6 +17,7 @@ import json
import logging
import time
import uuid
+import tempfile
from collections.abc import Callable
from dataclasses import dataclass, field
from datetime import datetime
@@ -383,7 +384,8 @@ class LoCoMoEvaluatorV2:
self._backend = self._config.backend_factory(self._current_user_id)
else:
# Default to LocalBackend
- db_path = self._config.db_path or f"/tmp/locomo_v2_{uuid.uuid4().hex[:8]}.db"
+ db_dir = tempfile.gettempdir()
+ db_path = self._config.db_path or f"{db_dir}/locomo_v2_{uuid.uuid4().hex[:8]}.db"
backend_config = LocalBackendConfig(db_path=db_path)
self._backend = LocalBackend(backend_config)
diff --git a/headroom/evals/memory/runner_v3.py b/headroom/evals/memory/runner_v3.py
index c066b5a06..91e49f421 100644
--- a/headroom/evals/memory/runner_v3.py
+++ b/headroom/evals/memory/runner_v3.py
@@ -22,6 +22,7 @@ import json
import logging
import time
import uuid
+import tempfile
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
@@ -261,7 +262,8 @@ class LoCoMoEvaluatorV3:
graph_status = "with graph" if self._config.mem0_enable_graph else "vector-only"
logger.info(f"Using Mem0 backend ({self._config.mem0_mode} mode, {graph_status})")
else:
- db_path = self._config.db_path or f"/tmp/locomo_v3_{uuid.uuid4().hex[:8]}.db"
+ db_dir = tempfile.gettempdir()
+ db_path = self._config.db_path or f"{db_dir}/locomo_v3_{uuid.uuid4().hex[:8]}.db"
self._backend = LocalBackend(LocalBackendConfig(db_path=db_path))
logger.info("Using LocalBackend")
diff --git a/headroom/memory/backends/direct_mem0.py b/headroom/memory/backends/direct_mem0.py
index 4c20b1537..7d07ec0de 100644
--- a/headroom/memory/backends/direct_mem0.py
+++ b/headroom/memory/backends/direct_mem0.py
@@ -275,7 +275,7 @@ class DirectMem0Adapter:
"user_id": user_id,
"importance": importance,
"created_at": _utcnow().isoformat(),
- "hash": hashlib.md5(fact.encode()).hexdigest(),
+ "hash": hashlib.md5(fact.encode()).hexdigest(), # nosec B324
**(metadata or {}),
}
diff --git a/headroom/parser.py b/headroom/parser.py
index fe93f9d77..970693547 100644
--- a/headroom/parser.py
+++ b/headroom/parser.py
@@ -33,7 +33,7 @@ RAG_PATTERN = re.compile("|".join(RAG_MARKERS), re.IGNORECASE)
def compute_hash(text: str) -> str:
"""Compute hash of text, truncated to 16 chars."""
- return hashlib.md5(text.encode()).hexdigest()[:16]
+ return hashlib.md5(text.encode()).hexdigest()[:16] # nosec B324
def detect_waste_signals(text: str, tokenizer: Tokenizer) -> WasteSignals:
diff --git a/headroom/prediction/feature_extractor.py b/headroom/prediction/feature_extractor.py
index a0222524e..afce6d87a 100644
--- a/headroom/prediction/feature_extractor.py
+++ b/headroom/prediction/feature_extractor.py
@@ -2138,7 +2138,7 @@ class MetaExtractor(BaseFeatureExtractor):
features.available_output_tokens = min(features.available_output_tokens, max_tokens)
# Prompt hash
- features.prompt_hash = hashlib.md5(text.encode()).hexdigest()[:16]
+ features.prompt_hash = hashlib.md5(text.encode()).hexdigest()[:16] # nosec B324
features.prompt_signature = self._compute_signature(text)
# Conversation features
@@ -2324,7 +2324,7 @@ class PromptFeatureExtractor:
PromptFeatures containing all extracted features.
"""
# Check cache
- cache_key = hashlib.md5(f"{prompt}:{model}:{system_prompt}".encode()).hexdigest()
+ cache_key = hashlib.md5(f"{prompt}:{model}:{system_prompt}".encode()).hexdigest() # nosec B324
if use_cache and cache_key in self._cache:
return self._cache[cache_key]
diff --git a/headroom/transforms/adaptive_sizer.py b/headroom/transforms/adaptive_sizer.py
index 51f3befbf..d8228e163 100644
--- a/headroom/transforms/adaptive_sizer.py
+++ b/headroom/transforms/adaptive_sizer.py
@@ -200,7 +200,7 @@ def _simhash(text: str) -> int:
# Character 4-grams
for i in range(max(1, len(text_lower) - 3)):
gram = text_lower[i : i + 4]
- h = int(hashlib.md5(gram.encode(), usedforsecurity=False).hexdigest()[:16], 16)
+ h = int(hashlib.md5(gram.encode(), usedforsecurity=False).hexdigest()[:16], 16) # nosec B324
for j in range(64):
if h & (1 << j):
v[j] += 1
diff --git a/headroom/transforms/anchor_selector.py b/headroom/transforms/anchor_selector.py
index d406e859f..d6bb60f9c 100644
--- a/headroom/transforms/anchor_selector.py
+++ b/headroom/transforms/anchor_selector.py
@@ -334,7 +334,7 @@ def compute_item_hash(item: dict[str, Any]) -> str:
content = json.dumps(item, sort_keys=True, default=str)
except (TypeError, ValueError):
content = str(item)
- return hashlib.md5(content.encode()).hexdigest()[:16]
+ return hashlib.md5(content.encode()).hexdigest()[:16] # nosec B324
class AnchorSelector:
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py
index 80c22dd8e..97c0e0891 100644
--- a/headroom/transforms/smart_crusher.py
+++ b/headroom/transforms/smart_crusher.py
@@ -1762,7 +1762,7 @@ class SmartCrusher(Transform):
else:
# Non-dict items: use string representation
content = str(item)
- item_hash = hashlib.md5(content.encode()).hexdigest()[:16]
+ item_hash = hashlib.md5(content.encode()).hexdigest()[:16] # nosec B324
except (TypeError, ValueError, RecursionError) as e:
# Serialization failed - keep the item (fail-safe)
logger.debug("Dedup hash failed for item at index %d: %s. Keeping item.", idx, e)
@@ -1832,7 +1832,7 @@ class SmartCrusher(Transform):
content = json.dumps(item, sort_keys=True, default=str)
else:
content = str(item)
- seen_hashes.add(hashlib.md5(content.encode()).hexdigest()[:16])
+ seen_hashes.add(hashlib.md5(content.encode()).hexdigest()[:16]) # nosec B324
except (TypeError, ValueError, RecursionError):
pass # Skip hash computation failures
@@ -1865,7 +1865,7 @@ class SmartCrusher(Transform):
content = json.dumps(item, sort_keys=True, default=str)
else:
content = str(item)
- item_hash = hashlib.md5(content.encode()).hexdigest()[:16]
+ item_hash = hashlib.md5(content.encode()).hexdigest()[:16] # nosec B324
except (TypeError, ValueError, RecursionError):
# Hash failure - use index as unique hash (fail-safe)
item_hash = f"__idx_{idx}__"
@@ -3336,7 +3336,7 @@ class SmartCrusher(Transform):
clusters: dict[str, list[int]] = {}
for i, item in enumerate(items):
msg = str(item.get(message_field, ""))[:50]
- msg_hash = hashlib.md5(msg.encode()).hexdigest()[:8]
+ msg_hash = hashlib.md5(msg.encode()).hexdigest()[:8] # nosec B324
if msg_hash not in clusters:
clusters[msg_hash] = []
clusters[msg_hash].append(i)
diff --git a/headroom/utils.py b/headroom/utils.py
index 72cd95e0e..4a559e48d 100644
--- a/headroom/utils.py
+++ b/headroom/utils.py
@@ -40,7 +40,7 @@ def fast_hash(data: str | bytes, length: int = 16) -> str:
"""
if isinstance(data, str):
data = data.encode("utf-8")
- return hashlib.md5(data).hexdigest()[:length]
+ return hashlib.md5(data).hexdigest()[:length] # nosec B324
def extract_user_query(messages: list[dict[str, Any]]) -> str:
diff --git a/tests/test_compression_store.py b/tests/test_compression_store.py
index 2e2f4b5a2..6a7c6b2bb 100644
--- a/tests/test_compression_store.py
+++ b/tests/test_compression_store.py
@@ -1252,7 +1252,7 @@ class TestHashCollisionDetection:
def test_hash_uses_md5_truncated(self, store: CompressionStore):
"""Hash is MD5 truncated to 24 characters (fast, non-crypto)."""
content = "test content"
- expected_hash = hashlib.md5(content.encode()).hexdigest()[:24]
+ expected_hash = hashlib.md5(content.encode()).hexdigest()[:24] # nosec B324
hash_key = store.store(original=content, compressed="[]")