Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

20.3 KB
Raw Blame
"""Tests for report_generator module."""

import re
from unittest.mock import MagicMock

import pytest

from local_deep_research.error_handling.report_generator import (
    ErrorReportGenerator,
)
from local_deep_research.error_handling.error_reporter import ErrorCategory


class TestErrorReportGeneratorInit:
    """Tests for ErrorReportGenerator initialization."""

    def test_initializes_error_reporter(self, error_report_generator):
        """Should initialize with ErrorReporter."""
        assert hasattr(error_report_generator, "error_reporter")
        assert error_report_generator.error_reporter is not None

    def test_accepts_optional_llm(self):
        """Should accept optional LLM (kept for compatibility)."""
        mock_llm = MagicMock()
        generator = ErrorReportGenerator(llm=mock_llm)
        # LLM is not used, just accepted
        assert generator is not None


class TestGenerateErrorReport:
    """Tests for generate_error_report method."""

    def test_returns_markdown_string(self, error_report_generator):
        """Should return markdown string."""
        result = error_report_generator.generate_error_report(
            "Connection refused", "test query"
        )
        assert isinstance(result, str)
        assert "#" in result  # Has markdown headers

    def test_includes_error_type(self, error_report_generator):
        """Should include error type in report."""
        result = error_report_generator.generate_error_report(
            "Connection refused", "test query"
        )
        assert "Error Type:" in result

    def test_includes_what_happened(self, error_report_generator):
        """Should include what happened section."""
        result = error_report_generator.generate_error_report(
            "Connection refused", "test query"
        )
        assert "What happened:" in result

    def test_includes_help_links(self, error_report_generator):
        """Should include help links."""
        result = error_report_generator.generate_error_report(
            "Connection refused", "test query"
        )
        assert "Get Help" in result
        assert "Wiki" in result
        assert "Discord" in result
        assert "GitHub Issues" in result

    def test_includes_partial_results_when_available(
        self, error_report_generator, sample_partial_results
    ):
        """Should include partial results when available."""
        result = error_report_generator.generate_error_report(
            "Connection refused",
            "test query",
            partial_results=sample_partial_results,
        )
        assert "Partial Results" in result

    def test_handles_no_partial_results(self, error_report_generator):
        """Should handle no partial results gracefully."""
        result = error_report_generator.generate_error_report(
            "Connection refused", "test query", partial_results=None
        )
        assert isinstance(result, str)
        # Should not include partial results section
        assert "Partial Results" not in result

    def test_includes_research_id_when_provided(self, error_report_generator):
        """Should accept research_id parameter."""
        result = error_report_generator.generate_error_report(
            "Error", "query", research_id="test-uuid"
        )
        assert isinstance(result, str)

    def test_includes_search_iterations_when_provided(
        self, error_report_generator
    ):
        """Should accept search_iterations parameter."""
        result = error_report_generator.generate_error_report(
            "Error", "query", search_iterations=5
        )
        assert isinstance(result, str)

    def test_fallback_on_exception(self, error_report_generator):
        """Should return fallback report on exception."""
        # Mock error_reporter to raise exception
        error_report_generator.error_reporter.analyze_error = MagicMock(
            side_effect=Exception("test error")
        )
        result = error_report_generator.generate_error_report("Error", "query")
        assert "Research Failed" in result
        assert "Error report generation failed" in result


# Where CommonMark raw HTML can begin: an open or closing tag, a comment, a
# declaration or a processing instruction always starts with "<" followed by
# a letter, "/", "!" or "?". Text inside an inline code span is literal, so
# a placeholder such as `ollama pull <model-name>` is not raw HTML.
_RAW_HTML_START = re.compile(r"<[A-Za-z/!?]")
_CODE_SPAN = re.compile(r"`[^`\n]*`")


def _raw_html_outside_code(report):
    """The first place raw HTML could begin outside an inline code span."""
    return _RAW_HTML_START.search(_CODE_SPAN.sub("", report))


class TestErrorReportIsPlainMarkdown:
    """The saved report is shown through renderMarkdown() (services/ui.js),
    which displays raw HTML in Markdown as literal text. The report's own
    structure must therefore be plain Markdown: with inputs that contain no
    HTML, the report contains none either."""

    def test_partial_results_are_a_markdown_section(
        self, error_report_generator, sample_partial_results
    ):
        """Regression: the partial results used to be wrapped in
        <details><summary>, which renders as escaped tags."""
        report = error_report_generator.generate_error_report(
            "Connection refused",
            "test query",
            partial_results=sample_partial_results,
        )

        assert "\n## 📊 Partial Results Available\n" in report
        assert "### Search Results Found" in report
        assert _raw_html_outside_code(report) is None, report

    @pytest.mark.parametrize(
        "error_message",
        [
            "Connection refused",
            "Model gemma2:9b not found in Ollama",
            "zzqx unexpected failure",
        ],
    )
    def test_report_without_partial_results_has_no_raw_html(
        self, error_report_generator, error_message
    ):
        report = error_report_generator.generate_error_report(
            error_message, "test query", partial_results=None
        )

        assert _raw_html_outside_code(report) is None, report

    def test_fallback_report_has_no_raw_html(self, error_report_generator):
        error_report_generator.error_reporter.analyze_error = MagicMock(
            side_effect=Exception("test error")
        )

        report = error_report_generator.generate_error_report("Error", "query")

        assert "Error report generation failed" in report
        assert _raw_html_outside_code(report) is None, report


class TestFormatPartialResults:
    """Tests for _format_partial_results method."""

    def test_returns_empty_for_none(self, error_report_generator):
        """Should return empty string for None."""
        result = error_report_generator._format_partial_results(None)
        assert result == ""

    def test_returns_empty_for_empty_dict(self, error_report_generator):
        """Should return empty string for empty dict."""
        result = error_report_generator._format_partial_results({})
        assert result == ""

    def test_formats_current_knowledge(self, error_report_generator):
        """Should format current_knowledge section."""
        partial = {"current_knowledge": "This is knowledge content " * 10}
        result = error_report_generator._format_partial_results(partial)
        assert "Research Summary" in result

    def test_truncates_long_knowledge(self, error_report_generator):
        """Should truncate knowledge over 1000 chars."""
        long_content = "x" * 1500
        partial = {"current_knowledge": long_content}
        result = error_report_generator._format_partial_results(partial)
        assert "..." in result

    def test_skips_short_knowledge(self, error_report_generator):
        """Should skip knowledge under 50 chars."""
        partial = {"current_knowledge": "short"}
        result = error_report_generator._format_partial_results(partial)
        assert "Research Summary" not in result

    def test_formats_search_results(self, error_report_generator):
        """Should format search_results section."""
        partial = {
            "search_results": [
                {"title": "Result 1", "url": "https://example.com/1"},
                {"title": "Result 2", "url": "https://example.com/2"},
            ]
        }
        result = error_report_generator._format_partial_results(partial)
        assert "Search Results Found" in result
        assert "Result 1" in result
        assert "https://example.com/1" in result

    def test_limits_search_results_to_five(self, error_report_generator):
        """Should show only top 5 search results."""
        partial = {
            "search_results": [
                {"title": f"Result {i}", "url": f"https://example.com/{i}"}
                for i in range(10)
            ]
        }
        result = error_report_generator._format_partial_results(partial)
        # Results are 0-indexed, so 5 results means indices 0-4
        assert "Result 4" in result
        assert "Result 5" not in result

    def test_formats_findings(self, error_report_generator):
        """Should format findings section."""
        partial = {
            "findings": [
                {"phase": "Phase 1", "content": "Finding content 1"},
                {"phase": "Phase 2", "content": "Finding content 2"},
            ]
        }
        result = error_report_generator._format_partial_results(partial)
        assert "Research Findings" in result
        assert "Phase 1" in result

    def test_skips_error_findings(self, error_report_generator):
        """Should skip findings that start with Error:."""
        partial = {
            "findings": [
                {"phase": "Phase 1", "content": "Error: Something went wrong"},
                {"phase": "Phase 2", "content": "Valid finding"},
            ]
        }
        result = error_report_generator._format_partial_results(partial)
        assert "Valid finding" in result
        # Error finding should be skipped

    def test_limits_findings_to_three(self, error_report_generator):
        """Should show only top 3 findings."""
        partial = {
            "findings": [
                {"phase": f"Phase {i}", "content": f"Finding {i}"}
                for i in range(5)
            ]
        }
        result = error_report_generator._format_partial_results(partial)
        assert "Phase 2" in result
        assert "Phase 4" not in result

    def test_truncates_long_finding_content(self, error_report_generator):
        """Should truncate finding content over 500 chars."""
        partial = {"findings": [{"phase": "Phase 1", "content": "x" * 600}]}
        result = error_report_generator._format_partial_results(partial)
        assert "..." in result

    def test_includes_note_when_results_exist(self, error_report_generator):
        """Should include note about successful collection."""
        partial = {
            "search_results": [
                {"title": "Result", "url": "https://example.com"}
            ]
        }
        result = error_report_generator._format_partial_results(partial)
        assert "successfully collected" in result


class TestGenerateQuickErrorSummary:
    """Tests for generate_quick_error_summary method."""

    def test_returns_dict(self, error_report_generator):
        """Should return dictionary."""
        result = error_report_generator.generate_quick_error_summary(
            "Connection refused"
        )
        assert isinstance(result, dict)

    def test_has_expected_keys(self, error_report_generator):
        """Should have expected keys."""
        result = error_report_generator.generate_quick_error_summary(
            "Connection refused"
        )
        expected_keys = ["title", "category", "severity", "recoverable"]
        for key in expected_keys:
            assert key in result

    def test_category_is_string_value(self, error_report_generator):
        """Should return category as string value."""
        result = error_report_generator.generate_quick_error_summary(
            "Connection refused"
        )
        assert isinstance(result["category"], str)

    def test_recoverable_is_boolean(self, error_report_generator):
        """Should return recoverable as boolean."""
        result = error_report_generator.generate_quick_error_summary(
            "Connection refused"
        )
        assert isinstance(result["recoverable"], bool)


class TestMakeErrorUserFriendly:
    """Tests for _make_error_user_friendly method."""

    def test_replaces_max_workers_error(self, error_report_generator):
        """Should replace max_workers error with friendly message."""
        result = error_report_generator._make_error_user_friendly(
            "max_workers must be greater than 0"
        )
        assert "LLM failed to generate" in result
        assert "Try this:" in result

    def test_replaces_ollama_eof_error(self, error_report_generator):
        """Should replace Ollama EOF error with friendly message."""
        result = error_report_generator._make_error_user_friendly(
            "POST predict encountered EOF"
        )
        assert "Lost connection to Ollama" in result

    def test_replaces_connection_refused(self, error_report_generator):
        """Should replace connection refused with friendly message."""
        result = error_report_generator._make_error_user_friendly(
            "Connection refused"
        )
        assert "Cannot connect to the LLM service" in result

    def test_replaces_search_too_long(self, error_report_generator):
        """Should replace search too long error."""
        result = error_report_generator._make_error_user_friendly(
            "The search is longer than 256 characters"
        )
        assert "search query is too long" in result

    def test_replaces_model_not_found(self, error_report_generator):
        """Should replace model not found error."""
        result = error_report_generator._make_error_user_friendly(
            "Model gemma not found in Ollama"
        )
        assert "model isn't available in Ollama" in result

    def test_replaces_api_key_error(self, error_report_generator):
        """Should replace API key errors."""
        result = error_report_generator._make_error_user_friendly(
            "No auth credentials found"
        )
        assert "API key is missing" in result

    def test_replaces_readonly_database(self, error_report_generator):
        """Should replace readonly database error."""
        result = error_report_generator._make_error_user_friendly(
            "Attempt to write readonly database"
        )
        assert "Permission issue" in result

    def test_replaces_docker_networking(self, error_report_generator):
        """Should replace Docker networking errors."""
        # Pattern requires "host.*localhost.*Docker" or similar
        result = error_report_generator._make_error_user_friendly(
            "Cannot connect to host at localhost via Docker"
        )
        # Should match docker pattern and suggest using host.docker.internal
        assert "Docker" in result or "host.docker.internal" in result

    def test_uses_generic_message_for_unknown(self, error_report_generator):
        """Should not expose the original message for unknown errors."""
        original = "Some random error that has no pattern"
        result = error_report_generator._make_error_user_friendly(original)
        assert "unexpected error" in result.lower()
        assert original not in result

    @pytest.mark.parametrize("prefix", ["", "Research failed: "])
    @pytest.mark.parametrize("separator", [" | Details: ", "\n"])
    def test_worker_message_with_added_detail_is_not_trusted(
        self, error_report_generator, prefix, separator
    ):
        marker = "/srv/private/provider-internals.txt"
        message = (
            f"{prefix}Authentication with the configured LLM provider failed."
            f"{separator}{marker}"
        )

        report = error_report_generator.generate_error_report(message, "query")

        assert marker not in report
        assert "unexpected error" in report.lower()

    @pytest.mark.parametrize(
        "message, expected",
        [
            (
                "Egress policy could not verify this run, so it was refused (fail-closed).",
                "Egress policy could not verify this run",
            ),
            (
                "Egress policy refused this run: a sensitive source would reach an exposing destination (sensitive_to_exposing_private_reason).",
                "a sensitive source would reach an exposing destination.",
            ),
            (
                "Egress policy refused this run (private_reason).",
                "Egress policy refused this run.",
            ),
        ],
    )
    def test_worker_policy_refusal_keeps_safe_guidance(
        self, error_report_generator, message, expected
    ):
        report = error_report_generator.generate_error_report(
            f"Research failed: {message}", "query"
        )

        assert expected in report
        assert "private_reason" not in report

    def test_omits_technical_error_from_replacement(
        self, error_report_generator
    ):
        """Should keep technical details out of replaced messages."""
        result = error_report_generator._make_error_user_friendly(
            "max_workers must be greater than 0"
        )
        assert "LLM failed to generate" in result
        assert "Technical error:" not in result
        assert "max_workers" not in result

    def test_case_insensitive_matching(self, error_report_generator):
        """Should match patterns case-insensitively."""
        result = error_report_generator._make_error_user_friendly(
            "CONNECTION REFUSED"
        )
        assert "Cannot connect" in result


class TestGetTechnicalContext:
    """Tests for _get_technical_context method."""

    def test_returns_empty_for_no_context(self, error_report_generator):
        """Should return empty string for no context."""
        error_analysis = {"category": ErrorCategory.CONNECTION_ERROR}
        result = error_report_generator._get_technical_context(
            error_analysis, None
        )
        # May still have category-based context
        assert isinstance(result, str)

    def test_includes_timing_info(self, error_report_generator):
        """Should include timing info when available."""
        error_analysis = {"category": ErrorCategory.CONNECTION_ERROR}
        partial = {
            "start_time": "2024-01-01T10:00:00",
            "last_activity": "2024-01-01T10:05:00",
        }
        result = error_report_generator._get_technical_context(
            error_analysis, partial
        )
        assert "Start Time:" in result

    def test_includes_model_info(self, error_report_generator):
        """Should include model info when available."""
        error_analysis = {"category": ErrorCategory.MODEL_ERROR}
        partial = {
            "model_config": {"model_name": "gpt-4", "provider": "openai"}
        }
        result = error_report_generator._get_technical_context(
            error_analysis, partial
        )
        assert "Model:" in result

    def test_includes_search_info(self, error_report_generator):
        """Should include search info when available."""
        error_analysis = {"category": ErrorCategory.SEARCH_ERROR}
        partial = {"search_config": {"engine": "duckduckgo", "max_results": 10}}
        result = error_report_generator._get_technical_context(
            error_analysis, partial
        )
        assert "Search Engine:" in result

    def test_includes_error_codes(self, error_report_generator):
        """Should include error codes when available."""
        error_analysis = {"category": ErrorCategory.CONNECTION_ERROR}
        partial = {"status_code": 503, "error_code": "SERVICE_UNAVAILABLE"}
        result = error_report_generator._get_technical_context(
            error_analysis, partial
        )
        assert "Status Code:" in result
        assert "Error Code:" in result

    def test_adds_category_specific_context(self, error_report_generator):
        """Should add category-specific context."""
        error_analysis = {"category": ErrorCategory.CONNECTION_ERROR}
        result = error_report_generator._get_technical_context(
            error_analysis, {}
        )
        assert "Network Error:" in result

        error_analysis = {"category": ErrorCategory.MODEL_ERROR}
        result = error_report_generator._get_technical_context(
            error_analysis, {}
        )
        assert "Model Error:" in result