Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

feat(rag): add standard-library jsonl loader (#8144)

Marc Bernardinoabb82030cbb4
diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/jsonl_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/jsonl_loader.py
new file mode 100644
index 000000000..8ab1e1cab
--- /dev/null
+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/jsonl_loader.py
@@ -0,0 +1,48 @@
+import json
+from typing import Any, NoReturn
+
+from crewai_tools.rag.base_loader import BaseLoader, LoaderResult
+from crewai_tools.rag.source_content import SourceContent
+
+
+class JSONLLoader(BaseLoader):
+ """Load a UTF-8 file or inline JSONL into one document.
+
+ Empty input produces empty content with a record count of zero.
+ Invalid records raise ValueError with their 1-based line number.
+ """
+
+ def load(self, content: SourceContent, **kwargs: Any) -> LoaderResult:
+ """Validate JSONL records and return their text, count, and source identity."""
+ source_ref = content.source_ref
+ text = content.source
+ if content.path_exists():
+ with open(source_ref, encoding="utf-8", newline="") as file:
+ text = file.read()
+
+ lines = text.split("\n")
+ if lines[-1] == "":
+ lines.pop()
+
+ records = []
+ for line_number, line in enumerate(lines, start=1):
+ try:
+ json.loads(line, parse_constant=self._reject_constant)
+ except ValueError as error:
+ raise ValueError(
+ f"Invalid JSONL at line {line_number}: {error}"
+ ) from error
+ records.append(line.strip(" \t\r"))
+
+ text = "\n".join(records)
+ return LoaderResult(
+ content=text,
+ source=source_ref,
+ metadata={"format": "jsonl", "record_count": len(records)},
+ doc_id=self.generate_doc_id(source_ref=source_ref, content=text),
+ )
+
+ @staticmethod
+ def _reject_constant(value: str) -> NoReturn:
+ """Reject NaN and infinity tokens, which are not valid JSON values."""
+ raise ValueError(f"Invalid JSON constant: {value}")
diff --git a/lib/crewai-tools/tests/rag/test_jsonl_loader.py b/lib/crewai-tools/tests/rag/test_jsonl_loader.py
new file mode 100644
index 000000000..2dba5b593
--- /dev/null
+++ b/lib/crewai-tools/tests/rag/test_jsonl_loader.py
@@ -0,0 +1,100 @@
+from pathlib import Path
+
+from crewai_tools.rag.loaders.jsonl_loader import JSONLLoader
+from crewai_tools.rag.source_content import SourceContent
+import pytest
+
+
[email protected]("from_file", [False, True])
+ "content, expected, count",
+ [
+ (
+ '{"ok":true}\n[1,null]\n"café"\n42\nfalse\nnull',
+ '{"ok":true}\n[1,null]\n"café"\n42\nfalse\nnull',
+ 6,
+ ),
+ ("1\n2", "1\n2", 2),
+ ("1\n2\n", "1\n2", 2),
+ ("1\r\n2", "1\n2", 2),
+ ("1\r\n2\r\n", "1\n2", 2),
+ ("", "", 0),
+ (" \t1\r\n 2\t", "1\n2", 2),
+ (
+ "1e400\n0.123456789012345678901234567890",
+ "1e400\n0.123456789012345678901234567890",
+ 2,
+ ),
+ ('"before\u2028after"', '"before\u2028after"', 1),
+ ('"a\ufeffb"', '"a\ufeffb"', 1),
+ (
+ '{"values":[true,null,"NaN","Infinity"]}',
+ '{"values":[true,null,"NaN","Infinity"]}',
+ 1,
+ ),
+ ],
+)
+def test_load_jsonl(
+ tmp_path: Path, from_file: bool, content: str, expected: str, count: int
+) -> None:
+ """Preserve valid records and derive consistent metadata for file or inline input."""
+ source = SourceContent(content)
+ if from_file:
+ path = tmp_path / "records.jsonl"
+ path.write_bytes(content.encode("utf-8"))
+ source = SourceContent(path)
+
+ result = JSONLLoader().load(source)
+
+ assert result.content == expected
+ assert result.metadata == {"format": "jsonl", "record_count": count}
+ assert result.source == (str(path) if from_file else source.source_ref)
+ assert result.doc_id == JSONLLoader.generate_doc_id(result.source, expected)
+ assert JSONLLoader().load(source).doc_id == result.doc_id
+
+
+def test_doc_id_changes_with_file_content(tmp_path: Path) -> None:
+ """Changing records at the same path must change the document ID."""
+ path = tmp_path / "records.jsonl"
+ path.write_text("1", encoding="utf-8")
+ source = SourceContent(path)
+ first = JSONLLoader().load(source)
+
+ path.write_text("2", encoding="utf-8")
+
+ assert JSONLLoader().load(source).doc_id != first.doc_id
+
+
[email protected]("from_file", [False, True])
+ "content, line_number",
+ [
+ ("1\n\n2", 2),
+ ("1\n \n2", 2),
+ ("1\n{bad}", 2),
+ ("1\n{} {}", 2),
+ ("\n", 1),
+ (" \t", 1),
+ ("\n1", 1),
+ ("1\n\n", 2),
+ ("1\r\n\r\n", 2),
+ ("1\nNaN", 2),
+ ("1\nInfinity", 2),
+ ("1\n-Infinity", 2),
+ ('1\n{"n":NaN}', 2),
+ ("1\n[Infinity]", 2),
+ ("\ufeff{}", 1),
+ ],
+)
+def test_invalid_jsonl_record(
+ tmp_path: Path, from_file: bool, content: str, line_number: int
+) -> None:
+ """Report the 1-based line containing a blank or invalid JSON record."""
+ source = SourceContent(content)
+ if from_file:
+ path = tmp_path / "invalid.jsonl"
+ path.write_bytes(content.encode("utf-8"))
+ source = SourceContent(path)
+
+ with pytest.raises(ValueError, match=rf"\bline {line_number}\b"):
+ JSONLLoader().load(source)