Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fix FileParserBlock decoding errors (#4399)
Marc Kelechava39bf3c2237ac
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 91d47a8e9..d9b87349f 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -22,6 +22,7 @@ import filetypeimport pandas as pdimport pyotpimport structlog+from charset_normalizer import from_bytesfrom email_validator import EmailNotValidError, validate_emailfrom jinja2 import StrictUndefinedfrom jinja2.sandbox import SandboxedEnvironment@@ -3037,10 +3038,36 @@ class FileParserBlock(Block):else:return FileType.CSV # Default to CSV for .csv and any other extensions+ def _detect_file_encoding(self, file_path: str) -> str:+ """Detect the encoding of a file using charset-normalizer with fallbacks.++ Reads a sample of the file (first 64KB) to detect encoding efficiently.+ Falls back through common encodings if detection fails.+ """+ sample_size = 65536 # 64KB sample for detection+ with open(file_path, "rb") as f:+ raw_data = f.read(sample_size)++ result = from_bytes(raw_data)+ best_match = result.best()+ if best_match and best_match.encoding:+ return best_match.encoding++ for encoding in ["utf-8", "cp1252", "latin-1"]:+ try:+ raw_data.decode(encoding)+ return encoding+ except UnicodeDecodeError:+ continue++ # latin-1 always succeeds (1:1 byte mapping), so this is a safety fallback+ return "latin-1"+def validate_file_type(self, file_url_used: str, file_path: str) -> None:if self.file_type == FileType.CSV:try:- with open(file_path) as file:+ encoding = self._detect_file_encoding(file_path)+ with open(file_path, encoding=encoding, errors="replace") as file:csv.Sniffer().sniff(file.read(1024))except csv.Error as e:raise InvalidFileType(file_url=file_url_used, file_type=self.file_type, error=str(e))@@ -3061,7 +3088,8 @@ class FileParserBlock(Block):async def _parse_csv_file(self, file_path: str) -> list[dict[str, Any]]:"""Parse CSV/TSV file and return list of dictionaries."""parsed_data = []- with open(file_path) as file:+ encoding = self._detect_file_encoding(file_path)+ with open(file_path, encoding=encoding, errors="replace") as file:# Try to detect the delimiter (comma for CSV, tab for TSV)sample = file.read(1024)file.seek(0) # Reset file pointer