Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat(rag): add standard-library jsonl loader (#8144)
Marc Bernardinoabb82030cbb4
diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/jsonl_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/jsonl_loader.pynew file mode 100644index 000000000..8ab1e1cab--- /dev/null+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/jsonl_loader.py@@ -0,0 +1,48 @@+import json+from typing import Any, NoReturn++from crewai_tools.rag.base_loader import BaseLoader, LoaderResult+from crewai_tools.rag.source_content import SourceContent+++class JSONLLoader(BaseLoader):+ """Load a UTF-8 file or inline JSONL into one document.++ Empty input produces empty content with a record count of zero.+ Invalid records raise ValueError with their 1-based line number.+ """++ def load(self, content: SourceContent, **kwargs: Any) -> LoaderResult:+ """Validate JSONL records and return their text, count, and source identity."""+ source_ref = content.source_ref+ text = content.source+ if content.path_exists():+ with open(source_ref, encoding="utf-8", newline="") as file:+ text = file.read()++ lines = text.split("\n")+ if lines[-1] == "":+ lines.pop()++ records = []+ for line_number, line in enumerate(lines, start=1):+ try:+ json.loads(line, parse_constant=self._reject_constant)+ except ValueError as error:+ raise ValueError(+ f"Invalid JSONL at line {line_number}: {error}"+ ) from error+ records.append(line.strip(" \t\r"))++ text = "\n".join(records)+ return LoaderResult(+ content=text,+ source=source_ref,+ metadata={"format": "jsonl", "record_count": len(records)},+ doc_id=self.generate_doc_id(source_ref=source_ref, content=text),+ )++ @staticmethod+ def _reject_constant(value: str) -> NoReturn:+ """Reject NaN and infinity tokens, which are not valid JSON values."""+ raise ValueError(f"Invalid JSON constant: {value}")diff --git a/lib/crewai-tools/tests/rag/test_jsonl_loader.py b/lib/crewai-tools/tests/rag/test_jsonl_loader.pynew file mode 100644index 000000000..2dba5b593--- /dev/null+++ b/lib/crewai-tools/tests/rag/test_jsonl_loader.py@@ -0,0 +1,100 @@+from pathlib import Path++from crewai_tools.rag.loaders.jsonl_loader import JSONLLoader+from crewai_tools.rag.source_content import SourceContent+import pytest++[email protected]("from_file", [False, True])+ "content, expected, count",+ [+ (+ '{"ok":true}\n[1,null]\n"café"\n42\nfalse\nnull',+ '{"ok":true}\n[1,null]\n"café"\n42\nfalse\nnull',+ 6,+ ),+ ("1\n2", "1\n2", 2),+ ("1\n2\n", "1\n2", 2),+ ("1\r\n2", "1\n2", 2),+ ("1\r\n2\r\n", "1\n2", 2),+ ("", "", 0),+ (" \t1\r\n 2\t", "1\n2", 2),+ (+ "1e400\n0.123456789012345678901234567890",+ "1e400\n0.123456789012345678901234567890",+ 2,+ ),+ ('"before\u2028after"', '"before\u2028after"', 1),+ ('"a\ufeffb"', '"a\ufeffb"', 1),+ (+ '{"values":[true,null,"NaN","Infinity"]}',+ '{"values":[true,null,"NaN","Infinity"]}',+ 1,+ ),+ ],+)+def test_load_jsonl(+ tmp_path: Path, from_file: bool, content: str, expected: str, count: int+) -> None:+ """Preserve valid records and derive consistent metadata for file or inline input."""+ source = SourceContent(content)+ if from_file:+ path = tmp_path / "records.jsonl"+ path.write_bytes(content.encode("utf-8"))+ source = SourceContent(path)++ result = JSONLLoader().load(source)++ assert result.content == expected+ assert result.metadata == {"format": "jsonl", "record_count": count}+ assert result.source == (str(path) if from_file else source.source_ref)+ assert result.doc_id == JSONLLoader.generate_doc_id(result.source, expected)+ assert JSONLLoader().load(source).doc_id == result.doc_id+++def test_doc_id_changes_with_file_content(tmp_path: Path) -> None:+ """Changing records at the same path must change the document ID."""+ path = tmp_path / "records.jsonl"+ path.write_text("1", encoding="utf-8")+ source = SourceContent(path)+ first = JSONLLoader().load(source)++ path.write_text("2", encoding="utf-8")++ assert JSONLLoader().load(source).doc_id != first.doc_id++[email protected]("from_file", [False, True])+ "content, line_number",+ [+ ("1\n\n2", 2),+ ("1\n \n2", 2),+ ("1\n{bad}", 2),+ ("1\n{} {}", 2),+ ("\n", 1),+ (" \t", 1),+ ("\n1", 1),+ ("1\n\n", 2),+ ("1\r\n\r\n", 2),+ ("1\nNaN", 2),+ ("1\nInfinity", 2),+ ("1\n-Infinity", 2),+ ('1\n{"n":NaN}', 2),+ ("1\n[Infinity]", 2),+ ("\ufeff{}", 1),+ ],+)+def test_invalid_jsonl_record(+ tmp_path: Path, from_file: bool, content: str, line_number: int+) -> None:+ """Report the 1-based line containing a blank or invalid JSON record."""+ source = SourceContent(content)+ if from_file:+ path = tmp_path / "invalid.jsonl"+ path.write_bytes(content.encode("utf-8"))+ source = SourceContent(path)++ with pytest.raises(ValueError, match=rf"\bline {line_number}\b"):+ JSONLLoader().load(source)