Skip to content

Self-improving AI Employees for everyone

24.1 KB
Raw Blame
"""Contract tests for document nodes: httpScraper, fileDownloader, documentParser,
textChunker, embeddingGenerator, vectorStore.

These tests freeze the input -> output behaviour documented in
`docs-internal/node-logic-flows/document/`. A refactor that breaks any of
these indicates the docs (and the user-visible contract) need to be updated
too.

External side-effects are mocked:
  - httpScraper / fileDownloader: httpx calls stubbed via respx.
  - documentParser: each parser library is patched at the module level
    (`services.handlers.document.<lib>`), or real HTML is fed through
    BeautifulSoup for the happy path.
  - textChunker: pure local function.
  - embeddingGenerator: native SDK clients are patched to return canned vectors.
  - vectorStore: the ChromaDB client is patched to a fake collection.
"""

from __future__ import annotations

import sys
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch

import httpx
import pytest
import respx


pytestmark = pytest.mark.node_contract


# ============================================================================
# httpScraper
# ============================================================================


class TestHttpScraper:
    @respx.mock
    async def test_happy_path_single_mode_extracts_links(self, harness):
        html = """
        <html><body>
          <a href="/file1.pdf">Doc 1</a>
          <a href="https://other.example/file2.pdf">Doc 2</a>
          <a href="/not-a-pdf.html">Skip</a>
        </body></html>
        """
        respx.get("https://example.com/list").mock(return_value=httpx.Response(200, text=html))

        result = await harness.execute(
            "httpScraper",
            {"url": "https://example.com/list", "link_selector": 'a[href$=".pdf"]'},
        )

        harness.assert_envelope(result, success=True)
        harness.assert_output_shape(result, ["items", "item_count", "errors"])
        payload = result["result"]
        assert payload["item_count"] == 2
        # urljoin resolves the relative link against the source URL
        urls = {i["url"] for i in payload["items"]}
        assert "https://example.com/file1.pdf" in urls
        assert "https://other.example/file2.pdf" in urls
        for item in payload["items"]:
            assert item["source_url"] == "https://example.com/list"
        assert payload["errors"] == []

    async def test_missing_url_fails_envelope(self, harness):
        result = await harness.execute("httpScraper", {"url": ""})
        harness.assert_envelope(result, success=False)
        assert "url is required" in result["error"].lower()

    @respx.mock
    async def test_per_url_http_error_collected_in_errors(self, harness):
        # Page iteration mode fetches two URLs; one 404s, the other succeeds.
        respx.get("https://example.com/p1").mock(return_value=httpx.Response(404))
        respx.get("https://example.com/p2").mock(return_value=httpx.Response(200, text='<a href="ok.pdf">OK</a>'))

        result = await harness.execute(
            "httpScraper",
            {
                "url": "https://example.com/p{page}",
                "iteration_mode": "page",
                "start_page": 1,
                "end_page": 2,
                "link_selector": "a",
            },
        )

        harness.assert_envelope(result, success=True)
        payload = result["result"]
        assert payload["item_count"] == 1
        assert payload["items"][0]["page"] == 2
        assert len(payload["errors"]) == 1
        assert "p1" in payload["errors"][0]

    @respx.mock
    async def test_date_mode_expands_range(self, harness):
        respx.get(url__regex=r"https://example\.com/\?d=\d{4}-\d{2}-\d{2}").mock(return_value=httpx.Response(200, text="<html></html>"))

        result = await harness.execute(
            "httpScraper",
            {
                "url": "https://example.com/?d={date}",
                "iteration_mode": "date",
                "start_date": "2025-01-01",
                "end_date": "2025-01-03",
                "link_selector": "a",
            },
        )

        harness.assert_envelope(result, success=True)
        # Three calls: 01, 02, 03
        assert len(respx.calls) == 3


# ============================================================================
# fileDownloader
# ============================================================================


class TestFileDownloader:
    @respx.mock
    async def test_happy_path_downloads_files(self, harness, tmp_path):
        respx.get("https://example.com/a.pdf").mock(return_value=httpx.Response(200, content=b"AAA"))
        respx.get("https://example.com/b.pdf").mock(return_value=httpx.Response(200, content=b"BBBB"))

        result = await harness.execute(
            "fileDownloader",
            {
                "items": [
                    {"url": "https://example.com/a.pdf"},
                    {"url": "https://example.com/b.pdf"},
                ],
                "outputDir": str(tmp_path),
                "maxWorkers": 2,
            },
        )

        harness.assert_envelope(result, success=True)
        harness.assert_output_shape(result, ["downloaded", "skipped", "failed", "files", "output_dir"])
        payload = result["result"]
        assert payload["downloaded"] == 2
        assert payload["failed"] == 0
        assert (tmp_path / "a.pdf").read_bytes() == b"AAA"
        assert (tmp_path / "b.pdf").read_bytes() == b"BBBB"

    async def test_empty_items_short_circuits(self, harness):
        result = await harness.execute("fileDownloader", {"items": []})
        harness.assert_envelope(result, success=True)
        payload = result["result"]
        assert (
            payload
            == {
                "downloaded": 0,
                "skipped": 0,
                "failed": 0,
                "files": [],
            }
            or payload["downloaded"] == 0
        )

    @respx.mock
    async def test_skip_existing_does_not_refetch(self, harness, tmp_path):
        existing = tmp_path / "a.pdf"
        existing.write_bytes(b"OLD")

        # No respx mock for a.pdf - if the handler tried to fetch, respx would raise.
        result = await harness.execute(
            "fileDownloader",
            {
                "items": [{"url": "https://example.com/a.pdf"}],
                "outputDir": str(tmp_path),
                "skipExisting": True,
            },
        )

        harness.assert_envelope(result, success=True)
        assert result["result"]["skipped"] == 1
        assert result["result"]["downloaded"] == 0
        # Existing bytes untouched
        assert existing.read_bytes() == b"OLD"

    @respx.mock
    async def test_http_error_counted_as_failed(self, harness, tmp_path):
        respx.get("https://example.com/bad.pdf").mock(return_value=httpx.Response(500, text="boom"))

        result = await harness.execute(
            "fileDownloader",
            {
                "items": [{"url": "https://example.com/bad.pdf"}],
                "outputDir": str(tmp_path),
            },
        )

        harness.assert_envelope(result, success=True)
        assert result["result"]["failed"] == 1
        assert result["result"]["downloaded"] == 0


# ============================================================================
# documentParser
# ============================================================================


class TestDocumentParser:
    async def test_beautifulsoup_parses_html_file(self, harness, tmp_path):
        html = "<html><body>" "<script>alert(1)</script>" "<style>.x {color:red}</style>" "<p>Hello <b>World</b></p>" "</body></html>"
        f = tmp_path / "page.html"
        f.write_text(html, encoding="utf-8")

        result = await harness.execute(
            "documentParser",
            {"file_path": str(f), "parser": "beautifulsoup"},
        )

        harness.assert_envelope(result, success=True)
        harness.assert_output_shape(result, ["documents", "parsed_count", "failed"])
        payload = result["result"]
        assert payload["parsed_count"] == 1
        doc = payload["documents"][0]
        assert doc["filename"] == "page.html"
        assert doc["parser"] == "beautifulsoup"
        # script and style tags stripped
        assert "alert(1)" not in doc["content"]
        assert ".x {color:red}" not in doc["content"]
        assert "Hello" in doc["content"] and "World" in doc["content"]

    async def test_no_files_returns_empty_documents(self, harness):
        result = await harness.execute("documentParser", {"files": [], "inputDir": ""})
        harness.assert_envelope(result, success=True)
        assert result["result"]["documents"] == []
        assert result["result"]["parsed_count"] == 0

    async def test_unknown_parser_collected_in_failed(self, harness, tmp_path):
        f = tmp_path / "x.txt"
        f.write_text("hello", encoding="utf-8")

        result = await harness.execute(
            "documentParser",
            {"files": [{"path": str(f)}], "parser": "nonsense"},
        )

        harness.assert_envelope(result, success=False)
        assert "invalid parameters" in result["error"].lower()

    async def test_pypdf_branch_via_patched_reader(self, harness, tmp_path):
        # Patch pypdf at its import site inside _parse_file_sync.
        f = tmp_path / "doc.pdf"
        f.write_bytes(b"%PDF-FAKE")

        fake_page = SimpleNamespace(extract_text=lambda: "page text")
        fake_reader = MagicMock()
        fake_reader.pages = [fake_page, fake_page]

        fake_pypdf = MagicMock()
        fake_pypdf.PdfReader = MagicMock(return_value=fake_reader)

        with patch.dict(sys.modules, {"pypdf": fake_pypdf}):
            result = await harness.execute(
                "documentParser",
                {"file_path": str(f), "parser": "pypdf"},
            )

        harness.assert_envelope(result, success=True)
        doc = result["result"]["documents"][0]
        assert doc["parser"] == "pypdf"
        assert "page text" in doc["content"]


# ============================================================================
# textChunker
# ============================================================================


class TestTextChunker:
    async def test_recursive_chunks_long_text(self, harness):
        content = "abcdefghij" * 200  # 2000 chars
        result = await harness.execute(
            "textChunker",
            {
                "documents": [{"content": content, "source": "src1"}],
                "chunk_size": 500,
                "chunk_overlap": 50,
                "strategy": "recursive",
            },
        )

        harness.assert_envelope(result, success=True)
        harness.assert_output_shape(result, ["chunks", "chunk_count"])
        payload = result["result"]
        assert payload["chunk_count"] >= 3  # 2000 chars / 500 with overlap
        # chunk_index is 0-based and monotonically increasing
        indices = [c["chunk_index"] for c in payload["chunks"]]
        assert indices == list(range(len(payload["chunks"])))
        # source preserved
        assert all(c["source"] == "src1" for c in payload["chunks"])

    async def test_empty_documents_short_circuits(self, harness):
        result = await harness.execute("textChunker", {"documents": []})
        harness.assert_envelope(result, success=True)
        assert result["result"] == {"chunks": [], "chunk_count": 0}

    async def test_empty_content_doc_is_skipped(self, harness):
        result = await harness.execute(
            "textChunker",
            {
                "documents": [
                    {"content": "", "source": "empty"},
                    {"content": "short text", "source": "s2"},
                ],
                "chunk_size": 1024,
                "chunk_overlap": 100,
            },
        )
        harness.assert_envelope(result, success=True)
        chunks = result["result"]["chunks"]
        # Only the non-empty doc contributed a chunk
        assert all(c["source"] == "s2" for c in chunks)
        assert len(chunks) == 1

    async def test_markdown_strategy_uses_markdown_splitter(self, harness):
        md = "# Heading\n\nParagraph one.\n\n## Sub\n\nParagraph two.\n"
        result = await harness.execute(
            "textChunker",
            {
                "documents": [{"content": md, "source": "md1"}],
                "chunk_size": 200,
                "chunk_overlap": 20,
                "strategy": "markdown",
            },
        )
        harness.assert_envelope(result, success=True)
        assert result["result"]["chunk_count"] >= 1


# ============================================================================
# embeddingGenerator
# ============================================================================


class TestEmbeddingGenerator:
    async def test_huggingface_happy_path_with_patched_embedder(self, harness):
        fake_embedder = MagicMock()
        fake_embedder.encode = MagicMock(
            return_value=[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]
        )

        fake_hf_mod = MagicMock()
        fake_hf_mod.SentenceTransformer = MagicMock(return_value=fake_embedder)

        with patch.dict(sys.modules, {"sentence_transformers": fake_hf_mod}):
            result = await harness.execute(
                "embeddingGenerator",
                {
                    "chunks": [
                        {"content": "first chunk", "source": "s1", "chunk_index": 0},
                        {"content": "second chunk", "source": "s1", "chunk_index": 1},
                    ],
                    "provider": "huggingface",
                    "model": "BAAI/bge-small-en-v1.5",
                },
            )

        harness.assert_envelope(result, success=True)
        harness.assert_output_shape(
            result,
            ["embeddings", "embedding_count", "dimensions", "chunks", "provider", "model"],
        )
        payload = result["result"]
        assert payload["embedding_count"] == 2
        assert payload["dimensions"] == 3
        assert payload["provider"] == "huggingface"
        # Embedder was constructed with the requested model
        fake_hf_mod.SentenceTransformer.assert_called_once_with(
            "BAAI/bge-small-en-v1.5"
        )
        fake_embedder.encode.assert_called_once()
        assert fake_embedder.encode.call_args.kwargs["batch_size"] == 32

    async def test_empty_chunks_short_circuits(self, harness):
        result = await harness.execute("embeddingGenerator", {"chunks": [], "provider": "huggingface"})
        harness.assert_envelope(result, success=True)
        payload = result["result"]
        assert payload["embedding_count"] == 0
        assert payload["dimensions"] == 0
        assert payload["embeddings"] == []

    async def test_unknown_provider_fails_envelope(self, harness):
        result = await harness.execute(
            "embeddingGenerator",
            {"chunks": [{"content": "x"}], "provider": "made_up_provider"},
        )
        harness.assert_envelope(result, success=False)
        assert "invalid parameters" in result["error"].lower()

    async def test_openai_provider_uses_apikey_param(self, harness):
        fake_client = MagicMock()
        fake_client.embeddings.create = AsyncMock(
            return_value=SimpleNamespace(
                data=[SimpleNamespace(index=0, embedding=[0.9, 0.8])]
            )
        )
        fake_client.close = AsyncMock()

        fake_openai_mod = MagicMock()
        fake_openai_mod.AsyncOpenAI = MagicMock(return_value=fake_client)

        with patch.dict(sys.modules, {"openai": fake_openai_mod}):
            result = await harness.execute(
                "embeddingGenerator",
                {
                    "chunks": [{"content": "only"}],
                    "provider": "openai",
                    "model": "text-embedding-3-small",
                    "api_key": "sk-test",
                },
            )

        harness.assert_envelope(result, success=True)
        fake_openai_mod.AsyncOpenAI.assert_called_once_with(api_key="sk-test")
        fake_client.embeddings.create.assert_awaited_once_with(
            model="text-embedding-3-small",
            input=["only"],
        )
        fake_client.close.assert_awaited_once()

    async def test_openai_provider_requires_api_key(self, harness):
        result = await harness.execute(
            "embeddingGenerator",
            {
                "chunks": [{"content": "only"}],
                "provider": "openai",
                "model": "text-embedding-3-small",
                "api_key": "",
            },
        )

        harness.assert_envelope(result, success=False)
        assert result.get("error_type") == "NodeUserError"
        assert "api key is required" in result["error"].lower()

    async def test_openai_sdk_failure_is_a_user_safe_node_error(
        self,
        harness,
    ):
        fake_client = MagicMock()
        fake_client.embeddings.create = AsyncMock(
            side_effect=RuntimeError("transport exploded"),
        )
        fake_client.close = AsyncMock()
        fake_openai_mod = MagicMock()
        fake_openai_mod.AsyncOpenAI = MagicMock(return_value=fake_client)

        with patch.dict(sys.modules, {"openai": fake_openai_mod}):
            result = await harness.execute(
                "embeddingGenerator",
                {
                    "chunks": [{"content": "only"}],
                    "provider": "openai",
                    "model": "text-embedding-3-small",
                    "api_key": "sk-test",
                },
            )

        harness.assert_envelope(result, success=False)
        assert result.get("error_type") == "NodeUserError"
        assert "openai embedding request failed" in result["error"].lower()
        assert "transport exploded" not in result["error"]
        fake_client.close.assert_awaited_once()

    async def test_ollama_provider_uses_native_batch_api(self, harness):
        fake_client = MagicMock()
        fake_client.embed = AsyncMock(
            return_value={"embeddings": [[0.2, 0.4], [0.6, 0.8]]}
        )
        fake_client.close = AsyncMock()
        fake_ollama_mod = MagicMock()
        fake_ollama_mod.AsyncClient = MagicMock(return_value=fake_client)

        with patch.dict(sys.modules, {"ollama": fake_ollama_mod}):
            result = await harness.execute(
                "embeddingGenerator",
                {
                    "chunks": [{"content": "one"}, {"content": "two"}],
                    "provider": "ollama",
                    "model": "nomic-embed-text",
                    "batch_size": 2,
                },
            )

        harness.assert_envelope(result, success=True)
        fake_client.embed.assert_awaited_once_with(
            model="nomic-embed-text",
            input=["one", "two"],
        )
        fake_client.close.assert_awaited_once()
        assert result["result"]["dimensions"] == 2


# ============================================================================
# vectorStore
# ============================================================================


class _FakeChromaCollection:
    def __init__(self):
        self.added = []
        self.deleted_ids = []
        self._count = 0

    def add(self, ids, embeddings, documents, metadatas):
        self.added.append({"ids": ids, "embeddings": embeddings, "documents": documents, "metadatas": metadatas})
        self._count += len(ids)

    def count(self):
        return self._count

    def query(self, query_embeddings, n_results):
        return {
            "ids": [["id-1", "id-2"]],
            "documents": [["doc one", "doc two"]],
            "metadatas": [[{"source": "s1"}, {"source": "s2"}]],
            "distances": [[0.1, 0.2]],
        }

    def delete(self, ids):
        self.deleted_ids.extend(ids)


def _patched_chromadb_module():
    coll = _FakeChromaCollection()
    client = MagicMock()
    client.get_or_create_collection = MagicMock(return_value=coll)

    module = MagicMock()
    module.PersistentClient = MagicMock(return_value=client)
    return module, coll


class TestVectorStore:
    async def test_chroma_store_happy_path(self, harness, tmp_path):
        chromadb_mod, coll = _patched_chromadb_module()

        with patch.dict(sys.modules, {"chromadb": chromadb_mod}):
            result = await harness.execute(
                "vectorStore",
                {
                    "operation": "store",
                    "backend": "chroma",
                    "collection_name": "docs",
                    "embeddings": [[0.1, 0.2], [0.3, 0.4]],
                    "chunks": [
                        {"content": "c1", "source": "s1", "chunk_index": 0},
                        {"content": "c2", "source": "s1", "chunk_index": 1},
                    ],
                    "persist_dir": str(tmp_path),
                },
            )

        harness.assert_envelope(result, success=True)
        payload = result["result"]
        assert payload["stored_count"] == 2
        assert payload["backend"] == "chroma"
        assert payload["collection_name"] == "docs"
        # The collection actually received our vectors + docs
        assert len(coll.added) == 1
        batch = coll.added[0]
        assert batch["embeddings"] == [[0.1, 0.2], [0.3, 0.4]]
        assert batch["documents"] == ["c1", "c2"]

    async def test_chroma_store_with_empty_embeddings_returns_zero(self, harness, tmp_path):
        chromadb_mod, coll = _patched_chromadb_module()
        with patch.dict(sys.modules, {"chromadb": chromadb_mod}):
            result = await harness.execute(
                "vectorStore",
                {
                    "operation": "store",
                    "backend": "chroma",
                    "embeddings": [],
                    "chunks": [],
                    "persist_dir": str(tmp_path),
                },
            )
        harness.assert_envelope(result, success=True)
        assert result["result"]["stored_count"] == 0
        # Nothing written to the underlying collection
        assert coll.added == []

    async def test_chroma_query_happy_path(self, harness, tmp_path):
        chromadb_mod, coll = _patched_chromadb_module()
        with patch.dict(sys.modules, {"chromadb": chromadb_mod}):
            result = await harness.execute(
                "vectorStore",
                {
                    "operation": "query",
                    "backend": "chroma",
                    "collection_name": "docs",
                    "query_embedding": [0.1, 0.2],
                    "top_k": 2,
                    "persist_dir": str(tmp_path),
                },
            )

        harness.assert_envelope(result, success=True)
        matches = result["result"]["matches"]
        assert len(matches) == 2
        assert matches[0]["id"] == "id-1"
        assert matches[0]["document"] == "doc one"
        assert matches[0]["distance"] == 0.1
        assert matches[0]["metadata"] == {"source": "s1"}

    async def test_chroma_delete_happy_path(self, harness, tmp_path):
        chromadb_mod, coll = _patched_chromadb_module()
        with patch.dict(sys.modules, {"chromadb": chromadb_mod}):
            result = await harness.execute(
                "vectorStore",
                {
                    "operation": "delete",
                    "backend": "chroma",
                    "ids": ["id-a", "id-b", "id-c"],
                    "persist_dir": str(tmp_path),
                },
            )

        harness.assert_envelope(result, success=True)
        assert result["result"]["deleted"] is True
        assert result["result"]["count"] == 3
        assert coll.deleted_ids == ["id-a", "id-b", "id-c"]

    async def test_unknown_backend_fails_envelope(self, harness):
        result = await harness.execute(
            "vectorStore",
            {"operation": "store", "backend": "madeup", "embeddings": [[0.1]]},
        )
        harness.assert_envelope(result, success=False)
        assert "invalid parameters" in result["error"].lower()

    async def test_pinecone_requires_api_key(self, harness):
        result = await harness.execute(
            "vectorStore",
            {
                "operation": "store",
                "backend": "pinecone",
                "embeddings": [[0.1]],
                "pineconeApiKey": "",
            },
        )
        harness.assert_envelope(result, success=False)
        assert "pinecone" in result["error"].lower() or "api key" in result["error"].lower()