OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Bad HTML filtering regexp
Olfa Maslah0fd7d5e82e56
diff --git a/src/main.py b/src/main.pyindex 46b3348a8..0630ec5fd 100644--- a/src/main.py+++ b/src/main.py@@ -8,6 +8,7 @@ import osimport reimport subprocessimport tempfile+from html.parser import HTMLParser# Configure structured logging earlyfrom connectors.langflow_connector_service import LangflowConnectorService@@ -98,6 +99,30 @@ logger = get_logger(__name__)EXCLUDED_INGESTION_FILES = {"warmup_ocr.pdf"}+class _VisibleTextHTMLParser(HTMLParser):+ """Extract visible text while skipping script/style content."""++ def __init__(self):+ super().__init__(convert_charrefs=True)+ self._ignored_depth = 0+ self._chunks: list[str] = []++ def handle_starttag(self, tag, attrs):+ if tag.lower() in {"script", "style"}:+ self._ignored_depth += 1++ def handle_endtag(self, tag):+ if tag.lower() in {"script", "style"} and self._ignored_depth > 0:+ self._ignored_depth -= 1++ def handle_data(self, data):+ if self._ignored_depth == 0 and data and not data.isspace():+ self._chunks.append(data)++ def get_text(self) -> str:+ return " ".join(self._chunks)++async def wait_for_opensearch():"""Wait for OpenSearch to be ready, delegating to the shared utility."""from utils.opensearch_utils import (@@ -561,24 +586,16 @@ async def _materialize_default_docs_url_as_text_file(raw_html = response.texttitle_match = re.search(- r"<title[^>]*>(.*?)</title>", raw_html, flags=re.IGNORECASE | re.DOTALL- )- title = html.unescape(title_match.group(1).strip()) if title_match else "OpenRAG"-- no_scripts = re.sub(- r"<script[^>]*>.*?</script>",- " ",+ r"<title[^>]*>(.*?)</title\s*>",raw_html,flags=re.IGNORECASE | re.DOTALL,)- no_styles = re.sub(- r"<style[^>]*>.*?</style>",- " ",- no_scripts,- flags=re.IGNORECASE | re.DOTALL,- )- text_only = re.sub(r"<[^>]+>", " ", no_styles)- normalized_text = re.sub(r"\s+", " ", html.unescape(text_only)).strip()+ title = html.unescape(title_match.group(1).strip()) if title_match else "OpenRAG"++ text_parser = _VisibleTextHTMLParser()+ text_parser.feed(raw_html)+ text_parser.close()+ normalized_text = re.sub(r"\s+", " ", text_parser.get_text()).strip()content = (f"{title}\n\n"