Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

3.3 KB
Raw Blame
import json
from typing import Any

from crewai_tools.rag.base_loader import BaseLoader, LoaderResult
from crewai_tools.rag.source_content import SourceContent


class NotebookLoader(BaseLoader):
    """Extract nbformat 4 cell sources from a UTF-8 file or inline JSON.

    Code is never executed. Outputs and attachments are omitted.
    Invalid cells raise ValueError with their 1-based cell number.
    """

    def load(self, content: SourceContent, **kwargs: Any) -> LoaderResult:
        """Return ordered cell source text, scalar metadata, and source identity."""
        source_ref = content.source_ref
        text = content.source
        if content.path_exists():
            with open(source_ref, encoding="utf-8-sig") as file:
                text = file.read()

        notebook = json.loads(text)
        if not isinstance(notebook, dict):
            raise ValueError("Expected a notebook object")
        version = notebook.get("nbformat")
        if not isinstance(version, int) or version != 4:
            raise ValueError("Only nbformat 4 notebooks are supported")
        cells = notebook.get("cells")
        if not isinstance(cells, list):
            raise ValueError("Notebook cells must be a list")
        metadata = self._extract_metadata(notebook)
        metadata.update(
            cell_count=len(cells), markdown_cells=0, code_cells=0, raw_cells=0
        )
        parts = []
        for number, cell in enumerate(cells, start=1):
            if not isinstance(cell, dict):
                raise ValueError(f"Invalid cell {number}: expected an object")
            cell_type = cell.get("cell_type")
            if cell_type not in ("markdown", "code", "raw"):
                raise ValueError(f"Invalid cell {number}: unsupported cell_type")
            source = cell.get("source")
            if isinstance(source, list) and all(
                isinstance(part, str) for part in source
            ):
                source = "".join(source)
            if not isinstance(source, str):
                raise ValueError(
                    f"Invalid cell {number}: source must be a string or list of strings"
                )
            parts.append(f"[cell {number}: {cell_type}]\n{source}")
            metadata[f"{cell_type}_cells"] += 1

        text = "\n\n".join(parts)
        return LoaderResult(
            content=text,
            source=source_ref,
            metadata=metadata,
            doc_id=self.generate_doc_id(source_ref=source_ref, content=text),
        )

    @staticmethod
    def _extract_metadata(notebook: dict[str, Any]) -> dict[str, Any]:
        """Retain notebook versions and valid language and kernel names."""
        metadata = {"format": "ipynb", "nbformat": notebook["nbformat"]}
        minor = notebook.get("nbformat_minor")
        if isinstance(minor, int) and not isinstance(minor, bool) and minor >= 0:
            metadata["nbformat_minor"] = minor
        notebook_metadata = notebook.get("metadata", {})
        if not isinstance(notebook_metadata, dict):
            return metadata
        for field, key in (("language_info", "language"), ("kernelspec", "kernel")):
            value = notebook_metadata.get(field)
            if isinstance(value, dict) and isinstance(value.get("name"), str):
                metadata[key] = value["name"]
        return metadata