A free open source IT asset/license management system
Refactor for codacy’s happiness (npath)
snipedf475008cdb0
diff --git a/app/Importer/Importer.php b/app/Importer/Importer.phpindex ff817ca09a..dab76b52fa 100644--- a/app/Importer/Importer.php+++ b/app/Importer/Importer.php@@ -197,19 +197,10 @@ abstract class Importer/*** Detect the source encoding of a non-UTF-8 string and convert it to- * UTF-8. Uses the Onnov detector first, falling back to mb_detect only- * when Onnov abstains (Onnov is usually more reliable when it commits- * to an answer, and overriding it with the CJK-leaning mb_detect list- * produces mojibake on short Cyrillic input).- *- * The iconv //IGNORE flag lets real-world CSVs with a stray invalid- * byte in an otherwise valid file import successfully. To keep that- * from turning into "silently accept entirely-corrupt input", the- * result is length-checked against the source: if the converted- * output is less than half the source size (rough proxy for- * "//IGNORE dropped most of the file"), we return the source- * unchanged rather than a nearly-empty buffer, so downstream sees the- * problem instead of an eerily-empty import.+ * UTF-8. Returns the source unchanged if the contents are already+ * UTF-8, if no source encoding can be determined, or if conversion+ * would drop more than half the source bytes (see+ * conversionExceedsLossThreshold).*/private static function convertToUtf8IfNeeded(string $contents): string{@@ -217,47 +208,93 @@ abstract class Importerreturn $contents;}+ $encoding = self::detectSourceEncoding($contents);+ if ($encoding === null) {+ return $contents;+ }++ $converted = self::runEncodingConversion($contents, $encoding);+ if ($converted === null) {+ return $contents;+ }++ if (self::conversionExceedsLossThreshold($contents, $converted, $encoding)) {+ return $contents;+ }++ return $converted;+ }++ /**+ * Try the Onnov detector first, falling back to mb_detect only when+ * Onnov abstains. Onnov is usually more reliable when it commits to+ * an answer, and overriding it with the CJK-leaning mb_detect list+ * produces mojibake on short Cyrillic input. Returns null when no+ * usable non-UTF-8 encoding was found.+ */+ private static function detectSourceEncoding(string $contents): ?string+ {$encoding = null;if (class_exists('\Onnov\DetectEncoding\EncodingDetector')) {$detector = new \Onnov\DetectEncoding\EncodingDetector;$encoding = $detector->getEncoding($contents);}+if (! $encoding || strcasecmp($encoding, 'UTF-8') === 0) {$detected = mb_detect_encoding($contents, ['UTF-8', 'GBK', 'GB2312', 'GB18030', 'BIG5', 'SJIS', 'EUC-JP', 'EUC-KR', 'Windows-1252', 'Windows-1251', 'ISO-8859-1'], true);if ($detected) {$encoding = $detected;}}+if (! $encoding || strcasecmp($encoding, 'UTF-8') === 0) {- return $contents;+ return null;}+ return $encoding;+ }++ /**+ * Run the actual byte-level conversion. Prefers iconv with //IGNORE so+ * real-world CSVs with a stray invalid byte still import successfully,+ * falling back to mb_convert_encoding on hosts without iconv. Returns+ * null when no converter is available or the converter refused the+ * input entirely.+ */+ private static function runEncodingConversion(string $contents, string $encoding): ?string+ {$converted = null;if (function_exists('iconv')) {- $converted = @iconv(strtoupper($encoding), 'UTF-8//IGNORE', $contents);- if ($converted === false) {- $converted = null;- }+ $result = @iconv(strtoupper($encoding), 'UTF-8//IGNORE', $contents);+ $converted = $result === false ? null : $result;} elseif (function_exists('mb_convert_encoding')) {$converted = @mb_convert_encoding($contents, 'UTF-8', $encoding);}- if ($converted === null || $converted === '') {- return $contents;- }-- if (strlen($converted) < intdiv(strlen($contents), 2)) {- Log::warning(sprintf(- 'CSV import: refusing lossy encoding conversion (%s -> UTF-8) that kept %d/%d bytes',- $encoding,- strlen($converted),- strlen($contents),- ));+ return ($converted === null || $converted === '') ? null : $converted;+ }- return $contents;+ /**+ * Loss-ratio safety net for the //IGNORE flag on iconv. If the+ * converted output is less than half the source size we treat that as+ * "//IGNORE dropped most of the file", log a warning, and let the+ * caller fall back to unconverted source so downstream sees the+ * problem instead of an eerily-empty import.+ */+ private static function conversionExceedsLossThreshold(string $source, string $converted, string $encoding): bool+ {+ if (strlen($converted) >= intdiv(strlen($source), 2)) {+ return false;}- return $converted;+ Log::warning(sprintf(+ 'CSV import: refusing lossy encoding conversion (%s -> UTF-8) that kept %d/%d bytes',+ $encoding,+ strlen($converted),+ strlen($source),+ ));++ return true;}// Cached Values for import lookups