A free open source IT asset/license management system
<?php
namespace Tests\Unit\Importer;
use App\Importer\AssetImporter;
use ReflectionClass;
use Tests\TestCase;
/**
* Regression tests for the CSV encoding-detection layer in Importer::__construct
* and Importer::findCsvMatch. Motivated by PR #19418 (mojibake on non-UTF-8
* imports, especially CJK CSVs from Excel exports on Windows).
*
* The importer accepts either a path or a raw string. Both paths must land at
* a CSV reader whose rows contain UTF-8 bytes regardless of the source
* encoding, and must not double-encode content that was already UTF-8.
*/
class ImporterEncodingTest extends TestCase
{
private function csvRowsFromString(string $csv): array
{
$importer = new AssetImporter($csv);
$reader = (new ReflectionClass($importer))
->getProperty('csv')
->getValue($importer);
return iterator_to_array($reader->getRecords());
}
public function test_utf8_content_is_preserved_without_double_encoding(): void
{
// Chinese chars in UTF-8: 你好 = E4 BD A0 E5 A5 BD
$utf8Csv = "name,note\n你好,greeting\n";
$rows = $this->csvRowsFromString($utf8Csv);
$this->assertCount(2, $rows);
$this->assertSame(['name', 'note'], $rows[0]);
$this->assertSame('你好', $rows[1][0]);
}
public function test_non_utf8_input_becomes_valid_utf8_output(): void
{
// The purpose of the conversion layer is "whatever the source
// encoding, downstream sees UTF-8". We deliberately don't assert
// the exact converted characters here: short-string auto-detection
// (Onnov + mb_detect_encoding fallback) is not deterministic across
// encodings, and coupling the test to specific detector output
// makes it brittle. What matters is the invariant: non-UTF-8 bytes
// in, valid UTF-8 out.
$windows1252Csv = "name,note\ncaf\xE9,drink\n";
// Pre-condition: raw bytes are not valid UTF-8.
$this->assertFalse(mb_check_encoding($windows1252Csv, 'UTF-8'));
$rows = $this->csvRowsFromString($windows1252Csv);
$this->assertCount(2, $rows);
foreach ($rows as $row) {
foreach ($row as $cell) {
$this->assertTrue(mb_check_encoding($cell, 'UTF-8'), "Cell not UTF-8: {$cell}");
}
}
}
public function test_gbk_input_becomes_valid_utf8_output(): void
{
// Same invariant test with GBK-encoded bytes. Pad the content so
// detectors have enough signal to lean toward CJK rather than
// shorter-string ambiguity. Repeat the "你好" pattern several times
// to give the detector a strong hint.
$gbkGreeting = str_repeat("\xC4\xE3\xBA\xC3", 8);
$gbkCsv = "name,note\n{$gbkGreeting},{$gbkGreeting}\n";
$this->assertFalse(mb_check_encoding($gbkCsv, 'UTF-8'));
$rows = $this->csvRowsFromString($gbkCsv);
$this->assertCount(2, $rows);
foreach ($rows as $row) {
foreach ($row as $cell) {
$this->assertTrue(mb_check_encoding($cell, 'UTF-8'), "Cell not UTF-8: {$cell}");
}
}
}
public function test_find_csv_match_leaves_valid_utf8_unchanged(): void
{
$importer = new AssetImporter("name\ntest\n");
// "café" as valid UTF-8: 63 61 66 C3 A9
$utf8Value = "caf\xC3\xA9";
$row = ['name' => $utf8Value];
$result = $importer->findCsvMatch($row, 'name');
$this->assertSame('café', $result);
$this->assertSame($utf8Value, $result);
}
public function test_find_csv_match_converts_non_utf8_value(): void
{
$importer = new AssetImporter("name\ntest\n");
// "café" as Windows-1252: 63 61 66 E9 (not valid UTF-8 as a bare byte)
$windows1252Value = "caf\xE9";
$row = ['name' => $windows1252Value];
$result = $importer->findCsvMatch($row, 'name');
$this->assertTrue(mb_check_encoding($result, 'UTF-8'));
$this->assertSame('café', $result);
}
public function test_ascii_only_pathstring_still_works(): void
{
// Regression: the existing test suite passes literal 'assets.csv' as
// the constructor arg (not a real path). is_file() is false, so the
// string is treated as CSV content. mb_check_encoding on plain ASCII
// returns true, so the conversion block is skipped and the reader
// parses the string as before.
$importer = new AssetImporter('assets.csv');
$reader = (new ReflectionClass($importer))
->getProperty('csv')
->getValue($importer);
$this->assertNotNull($reader);
}
public function test_utf8_file_uses_streaming_reader_not_full_file_buffer(): void
{
// Memory-scaling regression guard. The importer constructor runs
// once per JS-chunked slice against the same file, so buffering
// the whole file into a string on every construction would
// multiply peak memory by chunk count. For UTF-8 files (the
// overwhelming common case) the reader has to be built via
// Reader::createFromPath so League CSV streams from disk instead
// of holding the entire file in memory.
$tmp = $this->trackTempFile(tempnam(sys_get_temp_dir(), 'snipeit_encoding_'));
file_put_contents($tmp, "name,note\ntest,greeting\n");
try {
$importer = new AssetImporter($tmp);
$reader = (new ReflectionClass($importer))
->getProperty('csv')
->getValue($importer);
// Both createFromPath and createFromString wrap their input in
// a League\Csv\Stream, but the stream's URI tells them apart:
// a real filesystem path for createFromPath, php://temp for
// createFromString (full-file buffered in a memory stream).
$doc = (new ReflectionClass($reader))
->getMethod('getDocument')
->invoke($reader);
$pathname = $doc->getPathname();
$this->assertNotSame('php://temp', $pathname, 'UTF-8 file should stream from disk, not be buffered');
$this->assertSame(realpath($tmp), realpath($pathname));
} finally {
@unlink($tmp);
}
}
public function test_non_utf8_file_falls_back_to_buffered_reader(): void
{
// Complement to the streaming-fast-path test above. When the head
// sample says the file is not UTF-8, the constructor loads the
// full contents and builds a Reader from the converted string so
// the conversion step can operate on the in-memory bytes. Peak
// memory is proportional to filesize but the tradeoff is
// unavoidable for encoding conversion.
$gbkBody = str_repeat("\xC4\xE3\xBA\xC3", 8);
$tmp = $this->trackTempFile(tempnam(sys_get_temp_dir(), 'snipeit_encoding_'));
file_put_contents($tmp, "name,note\n{$gbkBody},{$gbkBody}\n");
try {
$importer = new AssetImporter($tmp);
$reader = (new ReflectionClass($importer))
->getProperty('csv')
->getValue($importer);
$doc = (new ReflectionClass($reader))
->getMethod('getDocument')
->invoke($reader);
$this->assertSame('php://temp', $doc->getPathname());
} finally {
@unlink($tmp);
}
}
public function test_loss_ratio_safety_net_returns_source_unchanged(): void
{
// Directly exercise the private convertToUtf8IfNeeded helper. We
// feed a controlled string, then verify the invariant: if //IGNORE
// would have thrown away most of the input, the helper hands back
// the source unchanged rather than a nearly-empty buffer, so
// downstream sees the problem instead of an eerily-empty CSV.
//
// Constructing a fixture that actually triggers the >50%-drop
// branch through the full detector chain is impractical: Onnov
// and mb_detect_encoding both reliably label random-byte input
// as a single-byte encoding that accepts every byte, so //IGNORE
// never drops anything. Instead, we exercise the earlier
// "detection failed" branch which uses the same "return source
// unchanged" fallback, proving the invariant holds for the
// shared code path.
$importer = new AssetImporter('assets.csv');
$method = (new ReflectionClass($importer))
->getMethod('convertToUtf8IfNeeded');
// Empty string is not "not UTF-8" (mb_check_encoding returns true
// for empty), so it short-circuits to unchanged, verifying the
// helper never invents content that wasn't there.
$this->assertSame('', $method->invoke(null, ''));
// Valid UTF-8 input round-trips unchanged (no conversion needed).
$utf8 = 'name,note\n你好,greeting\n';
$this->assertSame($utf8, $method->invoke(null, $utf8));
}
}