The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""Unit tests for the truncate-and-store web_extract path (no LLM).
Covers convert_base64_images_to_links, _truncate_with_footer, _store_full_text,
_get_extract_char_limit, and the end-to-end web_extract_tool truncation behavior.
"""
import asyncio
import json
import os
from pathlib import Path
from unittest.mock import patch
import pytest
import tools.web_tools as wt
from tools import web_tools_truncate
class TestImageConversion:
def test_markdown_base64_image_keeps_alt_drops_blob(self):
blob = "A" * 5000
text = f"before  after"
out = web_tools_truncate.convert_base64_images_to_links(text)
assert "[IMAGE: a cat]" in out
assert "base64" not in out
assert blob not in out
assert "before" in out and "after" in out
def test_bare_and_parenthesised_base64_become_placeholder(self):
blob = "Z" * 3000
bare = web_tools_truncate.convert_base64_images_to_links(f"data:image/gif;base64,{blob}")
assert bare == "[IMAGE]"
paren = web_tools_truncate.convert_base64_images_to_links(f"(data:image/gif;base64,{blob})")
assert paren == "[IMAGE]"
class TestTruncation:
def test_short_content_returned_whole(self):
content = "# Title\n\nshort body\n"
out, truncated = web_tools_truncate._truncate_with_footer(content, "https://e.com", 15000)
assert out == content
assert truncated is False
def test_truncation_stores_full_text_readable(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
body = "UNIQUE_MIDDLE_MARKER\n" + ("\n".join(f"row {i}" for i in range(5000)))
out, truncated = web_tools_truncate._truncate_with_footer(body, "https://example.com/doc", 3000)
assert truncated is True
# Extract the stored path from the footer and confirm full text is there.
path_line = next(ln for ln in out.splitlines() if "Full text saved to:" in ln)
stored_path = path_line.split("Full text saved to:", 1)[1].strip()
assert os.path.exists(stored_path)
full = Path(stored_path).read_text(encoding="utf-8")
assert "UNIQUE_MIDDLE_MARKER" in full
assert "row 2500" in full # the omitted-middle row is in the stored file
class TestCharLimitConfig:
def test_default_when_unset(self):
with patch("tools.web_tools._load_web_config", return_value={}):
assert web_tools_truncate._get_extract_char_limit() == web_tools_truncate.DEFAULT_EXTRACT_CHAR_LIMIT
def test_bad_value_falls_back(self):
with patch("tools.web_tools._load_web_config", return_value={"extract_char_limit": "nope"}):
assert web_tools_truncate._get_extract_char_limit() == web_tools_truncate.DEFAULT_EXTRACT_CHAR_LIMIT
class TestEndToEnd:
def test_web_extract_truncates_large_page_no_llm(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
big = "\n".join(f"para {i} " + "y" * 80 for i in range(3000))
class FakeProvider:
name = "fake"
display_name = "Fake"
def supports_extract(self):
return True
async def extract(self, urls, **kwargs):
return [{"url": urls[0], "title": "Big Page", "content": big,
"raw_content": big, "metadata": {}}]
with patch("tools.web_tools._ensure_web_plugins_loaded"), \
patch("tools.web_tools._get_extract_backend", return_value="fake"), \
patch("tools.web_tools.async_is_safe_url", new=_AsyncTrue()), \
patch("agent.web_search_registry.get_provider", return_value=FakeProvider()):
result = json.loads(asyncio.new_event_loop().run_until_complete(
wt.web_extract_tool(["https://example.com/big"], char_limit=5000)
))
assert "results" in result
content = result["results"][0]["content"]
assert "[TRUNCATED]" in content
assert "Full text saved to:" in content
# No LLM was involved: para 0 (head) and the last para (tail) are verbatim.
assert "para 0 " in content
assert "para 2999 " in content
def _make_awaitable(value):
async def _coro(*a, **k):
return value
return _coro()
class _AsyncTrue:
"""Async callable that always returns True (re-awaitable per call)."""
async def __call__(self, *a, **k):
return True
def test_binary_payload_is_refused_but_prose_with_short_signature_prefix_passes():
"""A backend that fetched a raw SQLite/zip file hands its bytes back as text; that must become an
error naming the type, while ordinary pages (even ones starting with 'BM' or 'MZ') pass untouched."""
results = [
{"url": "u", "raw_content": "SQLite format 3\x10\x01" + "x" * 5000}, # backend already dropped the NUL
{"url": "z", "raw_content": "PK\x03\x04" + "y" * 50},
{"url": "v", "raw_content": "BMW reviews are fine"},
{"url": "w", "raw_content": "# hi"},
]
web_tools_truncate._truncate_results(results, 5000, {"pages_truncated": 0, "truncation_metrics": []})
assert results[0]["content"] == "" and "SQLite database" in results[0]["error"]
assert results[1]["content"] == "" and "ZIP archive" in results[1]["error"]
assert results[2]["error"] is None if "error" in results[2] else True
assert results[2]["content"] == "BMW reviews are fine"
assert results[3]["content"] == "# hi"
def test_origin_error_page_keeps_its_error_and_stays_inside_the_budget():
body = "\n".join(f"error page line {i}" for i in range(4000))
results = [{"url": "https://example.com/forbidden", "content": body, "raw_content": body,
"error": "HTTP 403 from origin: Forbidden"}]
web_tools_truncate._truncate_results(results, 2000, {"pages_truncated": 0, "truncation_metrics": []})
assert results[0]["error"] == "HTTP 403 from origin: Forbidden"
assert "error page line 0" in results[0]["content"]
assert len(results[0]["content"]) < len(body) // 4