Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Error handling load_single_document() in ingest.py (#4852)
dcasota5528dd9d1170
diff --git a/examples/langchain-python-rag-privategpt/ingest.py b/examples/langchain-python-rag-privategpt/ingest.pyindex 35324775c..0f71ccf07 100755--- a/examples/langchain-python-rag-privategpt/ingest.py+++ b/examples/langchain-python-rag-privategpt/ingest.py@@ -77,13 +77,21 @@ LOADER_MAPPING = {def load_single_document(file_path: str) -> List[Document]:- ext = "." + file_path.rsplit(".", 1)[-1]- if ext in LOADER_MAPPING:- loader_class, loader_args = LOADER_MAPPING[ext]- loader = loader_class(file_path, **loader_args)- return loader.load()+ if os.path.getsize(file_path) != 0:+ filename, ext = os.path.splitext(file_path)+ if ext in LOADER_MAPPING:+ loader_class, loader_args = LOADER_MAPPING[ext]+ try:+ loader = loader_class(file_path, **loader_args)+ if loader:+ return loader.load()+ except:+ print(f"Corrupted file {file_path}. Ignoring it.")+ else:+ print(f"Unsupported file {file_path}. Ignoring it.")+ else:+ print(f"Empty file {file_path}. Ignoring it.")- raise ValueError(f"Unsupported file extension '{ext}'")def load_documents(source_dir: str, ignored_files: List[str] = []) -> List[Document]:"""@@ -100,7 +108,8 @@ def load_documents(source_dir: str, ignored_files: List[str] = []) -> List[Documresults = []with tqdm(total=len(filtered_files), desc='Loading new documents', ncols=80) as pbar:for i, docs in enumerate(pool.imap_unordered(load_single_document, filtered_files)):- results.extend(docs)+ if docs:+ results.extend(docs)pbar.update()return results