The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
community: add 'extract' mode to FireCrawlLoader for structured data extraction (#30242)
Bae-ChangHyund8510270eeb7
diff --git a/libs/community/langchain_community/document_loaders/firecrawl.py b/libs/community/langchain_community/document_loaders/firecrawl.pyindex 4423881dfe..12fdbbd560 100644--- a/libs/community/langchain_community/document_loaders/firecrawl.py+++ b/libs/community/langchain_community/document_loaders/firecrawl.py@@ -226,7 +226,7 @@ class FireCrawlLoader(BaseLoader):*,api_key: Optional[str] = None,api_url: Optional[str] = None,- mode: Literal["crawl", "scrape", "map"] = "crawl",+ mode: Literal["crawl", "scrape", "map", "extract"] = "crawl",params: Optional[dict] = None,):"""Initialize with API key and url.@@ -241,6 +241,7 @@ class FireCrawlLoader(BaseLoader):Options include "scrape" (single url),"crawl" (all accessible sub pages),"map" (returns list of links that are semantically related).+ "extract" (extracts structured data from a page).params: The parameters to pass to the Firecrawl API.Examples include crawlerOptions.For more details, visit: https://github.com/mendableai/firecrawl-py@@ -252,9 +253,10 @@ class FireCrawlLoader(BaseLoader):raise ImportError("`firecrawl` package not found, please run `pip install firecrawl-py`")- if mode not in ("crawl", "scrape", "search", "map"):+ if mode not in ("crawl", "scrape", "search", "map", "extract"):raise ValueError(- f"Invalid mode '{mode}'. Allowed: 'crawl', 'scrape', 'search', 'map'."+ f"""Invalid mode '{mode}'.+ Allowed: 'crawl', 'scrape', 'search', 'map', 'extract'.""")if not url:@@ -284,16 +286,23 @@ class FireCrawlLoader(BaseLoader):if not self.url:raise ValueError("URL is required for map mode")firecrawl_docs = self.firecrawl.map_url(self.url, params=self.params)+ elif self.mode == "extract":+ if not self.url:+ raise ValueError("URL is required for extract mode")+ firecrawl_docs = [+ str(self.firecrawl.extract([self.url], params=self.params))+ ]elif self.mode == "search":raise ValueError("Search mode is not supported in this version, please downgrade.")else:raise ValueError(- f"Invalid mode '{self.mode}'. Allowed: 'crawl', 'scrape', 'map'."+ f"""Invalid mode '{self.mode}'.+ Allowed: 'crawl', 'scrape', 'map', 'extract'.""")for doc in firecrawl_docs:- if self.mode == "map":+ if self.mode == "map" or self.mode == "extract":page_content = docmetadata = {}else:diff --git a/libs/community/tests/unit_tests/document_loaders/test_firecrawl.py b/libs/community/tests/unit_tests/document_loaders/test_firecrawl.pynew file mode 100644index 0000000000..11f285498a--- /dev/null+++ b/libs/community/tests/unit_tests/document_loaders/test_firecrawl.py@@ -0,0 +1,100 @@+"""Test FireCrawlLoader."""++import sys+from typing import Generator, List, Tuple+from unittest.mock import MagicMock++import pytest+from langchain_core.documents import Document++from langchain_community.document_loaders import FireCrawlLoader+++# firecrawl 모듈을 모킹하여 sys.modules에 등록[email protected](autouse=True)+def mock_firecrawl() -> Generator[Tuple[MagicMock, MagicMock], None, None]:+ """Mock firecrawl module for all tests."""+ mock_module = MagicMock()+ mock_client = MagicMock()+ # FirecrawlApp 클래스로 수정+ mock_module.FirecrawlApp.return_value = mock_client++ # extract 메서드의 반환값 설정+ response_dict = {+ "success": True,+ "data": {+ "title": "extracted title",+ "main contents": "extracted main contents",+ },+ "status": "completed",+ "expiresAt": "2025-03-12T12:42:09.000Z",+ }+ mock_client.extract.return_value = response_dict++ # sys.modules에 모의 모듈 삽입+ sys.modules["firecrawl"] = mock_module+ yield mock_module, mock_client # 테스트에서 필요할 경우 접근할 수 있도록 yield++ # 테스트 후 정리+ if "firecrawl" in sys.modules:+ del sys.modules["firecrawl"]+++class TestFireCrawlLoader:+ """Test FireCrawlLoader."""++ def test_load_extract_mode(+ self, mock_firecrawl: Tuple[MagicMock, MagicMock]+ ) -> List[Document]:+ """Test loading in extract mode."""+ # fixture에서 모킹된 객체 가져오기+ _, mock_client = mock_firecrawl++ params = {+ "prompt": "extract the title and main contents(write your own prompt here)",+ "schema": {+ "type": "object",+ "properties": {+ "title": {"type": "string"},+ "main contents": {"type": "string"},+ },+ "required": ["title", "main contents"],+ },+ "enableWebSearch": False,+ "ignoreSitemap": False,+ "showSources": False,+ "scrapeOptions": {+ "formats": ["markdown"],+ "onlyMainContent": True,+ "headers": {},+ "waitFor": 0,+ "mobile": False,+ "skipTlsVerification": False,+ "timeout": 30000,+ "removeBase64Images": True,+ "blockAds": True,+ "proxy": "basic",+ },+ }++ # FireCrawlLoader 인스턴스 생성 및 실행+ loader = FireCrawlLoader(+ url="https://example.com", api_key="fake-key", mode="extract", params=params+ )+ docs = list(loader.lazy_load()) # lazy_load 메서드 호출++ # 검증+ assert len(docs) == 1+ assert isinstance(docs[0].page_content, str)++ # extract 메서드가 올바른 인자로 호출되었는지 확인+ mock_client.extract.assert_called_once_with(+ ["https://example.com"], params=params+ )++ # 응답이 문자열로 변환되었으므로 각 속성이 문자열에 포함되어 있는지 확인+ assert "extracted title" in docs[0].page_content+ assert "extracted main contents" in docs[0].page_content+ assert "success" in docs[0].page_content++ return docs