Free, open-source file converter. Convert 263 formats offline by drag & drop — images, PDF, Word, Excel, PowerPoint, HWP/HWPX, video, audio, e-books and archives. No upload, no Office needed.
C# audio-converteravaloniabatch-conversionconvertercsharpdocument-converterdocx-to-pdfdotnetdrag-and-dropfile-conversionfile-converterheic-converterhwphwpximage-converterlinuxmacospdf-convertervideo-converterwindows
FIT: keep memory near the file size and read fields only in their profile width
SeoJCalive4e03c7fef35c
diff --git a/src/Filee.Engines/Geo/FitFormat.cs b/src/Filee.Engines/Geo/FitFormat.csindex 9fe9f3b9..59057986 100644--- a/src/Filee.Engines/Geo/FitFormat.cs+++ b/src/Filee.Engines/Geo/FitFormat.cs@@ -136,6 +136,7 @@ internal static class FitFormatuint? lastTime = null;int? fileType = null;var fileIdSeen = false;+ var recordCount = 0;while (position < limit){ct.ThrowIfCancellationRequested();@@ -194,7 +195,9 @@ internal static class FitFormatRequire(field.Size);var value = ReadNumber(bytes.AsSpan(position, field.Size), field.Type, definition.BigEndian);position += field.Size;- if (value is null)+ // A field read is taken only in its profile width as an integer: a float or wider type could carry+ // any size of value past the range the profile allows.+ if (value is null || ProfileWidth(definition.Message, field.Number) is { } width && IntegerWidth(field.Type) != width)continue;if (field.Number == 253 && value is >= 0x10000000 and < 0xFFFFFFFF){@@ -227,12 +230,18 @@ internal static class FitFormatcase 0: fileIdSeen = true; break;case 18: activity.AddSession(ToDate(startTime), sport); break;case 20:- if (activity.Records.Count >= maxRecords)+ if (recordCount++ >= maxRecords)throw new ConversionRefusedException("error.fit_too_large", "2000000 records");if (lat is < -90 or > 90 || lon is < -180 or > 180 || lat is null || lon is null)lat = lon = null;- activity.Records.Add(new FitRecord(ToDate(time), lat, lon, enhancedHeight ?? height,- heartRate, cadence, power, enhancedSpeed ?? speed, distance));+ var elevation = enhancedHeight ?? height;+ var recordSpeed = enhancedSpeed ?? speed;+ // A record without a single value adds nothing to any output, so it is counted but not kept.+ if (time is null && lat is null && elevation is null && heartRate is null && cadence is null+ && power is null && recordSpeed is null && distance is null)+ break;+ activity.Records.Add(new FitRecord(ToDate(time), lat, lon, elevation,+ heartRate, cadence, power, recordSpeed, distance));break;}}@@ -251,6 +260,25 @@ internal static class FitFormat}}+ /// <summary>The integer width the FIT Profile gives each field read here; other fields aren't width-checked.</summary>+ private static int? ProfileWidth(int message, int number) => (message, number) switch+ {+ (_, 253) or (18, 2) or (20, 0) or (20, 1) or (20, 5) or (20, 73) or (20, 78) => 4,+ (20, 2) or (20, 6) or (20, 7) => 2,+ (0, 0) or (18, 5) or (20, 3) or (20, 4) => 1,+ _ => null,+ };++ /// <summary>The width of an integer base type (enum, signed, unsigned and their invalid-zero kinds); 0 otherwise.</summary>+ private static int IntegerWidth(byte baseType) => (baseType & 0x1F) switch+ {+ 0 or 1 or 2 or 10 => 1,+ 3 or 4 or 11 => 2,+ 5 or 6 or 12 => 4,+ 14 or 15 or 16 => 8,+ _ => 0,+ };+private static DateTime? ToDate(uint? seconds) =>seconds is { } value ? DateTime.UnixEpoch.AddSeconds(value + FitEpochOffset) : null;@@ -306,17 +334,19 @@ internal static class FitFormat/// <summary>Writes one nonempty record per CSV row, keeping file order and empty sensor cells.</summary>public static void WriteCsv(FitActivity activity, string path){- var rows = new List<List<string>>- {- new() { "time", "longitude", "latitude", "elevation", "heart_rate", "cadence", "power", "speed", "distance" },- };- foreach (var record in activity.Records)- {- if (record.Time is null && record.Latitude is null && record.Longitude is null- && record.HeartRate is null && record.Cadence is null && record.Power is null- && record.Speed is null && record.Distance is null)- continue;- rows.Add([+ if (!activity.Records.Any(HasRow))+ throw new ConversionRefusedException("error.fit_no_records");+ string[] header = ["time", "longitude", "latitude", "elevation", "heart_rate", "cadence", "power", "speed", "distance"];+ // Rows are made as they are written, so a long activity never holds its whole CSV as text.+ CsvFormat.WriteRows(activity.Records.Where(HasRow).Select(Row).Prepend(header), path);++ static bool HasRow(FitRecord record) =>+ record.Time is not null || record.Latitude is not null || record.Longitude is not null+ || record.HeartRate is not null || record.Cadence is not null || record.Power is not null+ || record.Speed is not null || record.Distance is not null;++ static string[] Row(FitRecord record) =>+ [record.Time?.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture) ?? "",record.Longitude is { } lon ? GeoText.Degrees(lon) : "",record.Latitude is { } lat ? GeoText.Degrees(lat) : "",@@ -326,11 +356,7 @@ internal static class FitFormatrecord.Power?.ToString(CultureInfo.InvariantCulture) ?? "",record.Speed is { } speed ? GeoText.Number(speed) : "",record.Distance is { } distance ? GeoText.Number(distance) : "",- ]);- }- if (rows.Count == 1)- throw new ConversionRefusedException("error.fit_no_records");- CsvFormat.WriteRows(rows, path);+ ];}private static ConversionRefusedException Damaged(string detail) => new("error.fit_damaged", detail);diff --git a/src/Filee.Engines/Office/Sheets/CsvFormat.cs b/src/Filee.Engines/Office/Sheets/CsvFormat.csindex 4b4fc700..2cc65a62 100644--- a/src/Filee.Engines/Office/Sheets/CsvFormat.cs+++ b/src/Filee.Engines/Office/Sheets/CsvFormat.cs@@ -70,13 +70,32 @@ internal static class CsvFormatFile.WriteAllText(path, sb.ToString(), new UTF8Encoding(encoderShouldEmitUTF8Identifier: true));}- /// <summary>Writes rows of text as CSV the same way as <see cref="Write"/> (UTF-8 with BOM, CRLF).</summary>+ /// <summary>Writes rows of text as CSV the same way as <see cref="Write"/> (UTF-8 with BOM, CRLF), one row at a+ /// time, so a lazily made sequence is never held as text. The rows go to a file beside the target, which replaces+ /// it only once every row is written: a row that fails to come leaves a file already at the path as it was.</summary>public static void WriteRows(IEnumerable<IEnumerable<string>> rows, string path, char delimiter = ','){- var sb = new StringBuilder();- foreach (var row in rows)- sb.Append(string.Join(delimiter, row.Select(value => Quote(value, delimiter)))).Append("\r\n");- File.WriteAllText(path, sb.ToString(), new UTF8Encoding(encoderShouldEmitUTF8Identifier: true));+ // Replacing a file needs only the folder's permission on Unix: refuse what writing into the file would refuse.+ if (File.Exists(path))+ using (new FileStream(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) { }+ var partial = Path.Combine(Path.GetDirectoryName(Path.GetFullPath(path))!, $".{Path.GetFileName(path)}.{Guid.NewGuid():N}.tmp");+ try+ {+ using (var writer = new StreamWriter(partial, append: false, new UTF8Encoding(encoderShouldEmitUTF8Identifier: true)))+ {+ foreach (var row in rows)+ {+ writer.Write(string.Join(delimiter, row.Select(value => Quote(value, delimiter))));+ writer.Write("\r\n");+ }+ }+ File.Move(partial, path, overwrite: true);+ }+ catch+ {+ File.Delete(partial);+ throw;+ }}internal static List<List<string>> Parse(string text, char delimiter)diff --git a/tests/Filee.Engines.Tests/FitFormatTests.cs b/tests/Filee.Engines.Tests/FitFormatTests.csindex 59e9e16f..1776f36d 100644--- a/tests/Filee.Engines.Tests/FitFormatTests.cs+++ b/tests/Filee.Engines.Tests/FitFormatTests.cs@@ -133,6 +133,26 @@ public class FitFormatTests(EngineFixture fx) : IClassFixture<EngineFixture>], File.ReadAllLines((await fx.ConvertOkAsync(input, "csv")).Single()));}+ [Fact]+ public async Task Fields_not_in_their_profile_integer_width_are_left_out()+ {+ // Height and heart rate as float32 or a wider integer would carry values far past what the profile allows.+ FitBuilder.FitField[] fields =+ [+ FitBuilder.U32(253, First), FitBuilder.S32(0, 0x20000000), FitBuilder.S32(1, 0x40000000),+ new(2, 0x88, BitConverter.GetBytes(3e38f)), FitBuilder.U32(3, 4_000_000_000), new(4, 0x88, BitConverter.GetBytes(1e30f)),+ ];+ var input = Save(new FitBuilder().FileId().Define(0, 20, fields: fields).Message(0, fields));++ var gpx = (await fx.ConvertOkAsync(input, "gpx")).Single();+ Assert.Null(Child(Assert.Single(Elements(gpx, "wpt")), "ele"));+ Assert.Equal(+ [+ "time,longitude,latitude,elevation,heart_rate,cadence,power,speed,distance",+ $"{Stamp(First)},90,45,,,,,,",+ ], File.ReadAllLines((await fx.ConvertOkAsync(input, "csv")).Single()));+ }+[Fact]public async Task Single_position_is_a_point_in_each_spatial_format(){diff --git a/tests/Filee.Engines.Tests/FitReaderTests.cs b/tests/Filee.Engines.Tests/FitReaderTests.csindex 439fd7fa..17a58e42 100644--- a/tests/Filee.Engines.Tests/FitReaderTests.cs+++ b/tests/Filee.Engines.Tests/FitReaderTests.cs@@ -179,6 +179,25 @@ public class FitReaderTests(EngineFixture fx) : IClassFixture<EngineFixture>Assert.Equal(["running", "cycling"], activity.Sports);}+ [Fact]+ public void Empty_records_count_toward_the_limit_but_are_not_kept()+ {+ var time = new[] { FitBuilder.U32(253, 0x1000003B) };+ var builder = new FitBuilder().FileId().Define(1, 20, fields: time).Message(1, time).Define(0, 20);+ for (var i = 0; i < 500_000; i++)+ builder.Message(0, []);+ var path = FitBuilder.Save(fx.NewFolder(), builder.Build());+ // Warm the parser so the measurement holds only this file: its bytes, and no object per empty record.+ _ = Read(new FitBuilder().FileId().Define(1, 20, fields: time).Message(1, time));+ var before = GC.GetAllocatedBytesForCurrentThread();+ var activity = FitFormat.Read(path, CancellationToken.None);+ var allocated = GC.GetAllocatedBytesForCurrentThread() - before;+ Assert.True(allocated < new FileInfo(path).Length + 1_000_000, $"{allocated} bytes for a {new FileInfo(path).Length}-byte file");+ Assert.Equal(Time(0x1000003B), Assert.Single(activity.Records).Time);+ var error = Assert.Throws<ConversionRefusedException>(() => FitFormat.Read(path, CancellationToken.None, maxRecords: 500_000));+ Assert.Equal("error.fit_too_large", error.ErrorKey);+ }+[Fact]public void Configurable_file_and_record_limits_refuse_only_excess(){@@ -246,8 +265,8 @@ public class FitReaderTests(EngineFixture fx) : IClassFixture<EngineFixture>.Define(0, 20, fields: fields).Message(0, fields).Define(1, 20, fields: empty).Compressed(1, 0, empty).Compressed(1, 1, empty).Message(0, absolute).Message(0, fields).Compressed(1, 0x1D, empty));- Assert.Equal([null, null, null, Time(0x1000003B), null, Time(0x1000003D)],- activity.Records.Select(r => r.Time));+ // Records left without any value aren't kept, so a relative timestamp that became absolute would show up here.+ Assert.Equal([Time(0x1000003B), Time(0x1000003D)], activity.Records.Select(r => r.Time));}[Fact]diff --git a/tests/Filee.Engines.Tests/OfficeTests.cs b/tests/Filee.Engines.Tests/OfficeTests.csindex 5ff535cb..564ff3b1 100644--- a/tests/Filee.Engines.Tests/OfficeTests.cs+++ b/tests/Filee.Engines.Tests/OfficeTests.cs@@ -144,6 +144,40 @@ public class OfficeTests(EngineFixture fx) : IClassFixture<EngineFixture>Assert.Equal("90", sheet.Get(1, 1)!.Text);}+ [Fact]+ public void Csv_rows_that_fail_part_way_or_meet_a_read_only_file_leave_it_as_it_was()+ {+ var dir = fx.NewFolder();+ var csv = Path.Combine(dir, "rows.csv");+ File.WriteAllText(csv, "kept");++ Assert.Throws<IOException>(() => CsvFormat.WriteRows(FailingRows(), csv));+ Assert.Equal("kept", File.ReadAllText(csv));+ Assert.Equal([csv], Directory.GetFiles(dir));++ File.SetAttributes(csv, FileAttributes.ReadOnly);+ try+ {+ Assert.Throws<UnauthorizedAccessException>(() => CsvFormat.WriteRows([["a"]], csv));+ Assert.Equal("kept", File.ReadAllText(csv));+ Assert.Equal([csv], Directory.GetFiles(dir));+ }+ finally+ {+ File.SetAttributes(csv, FileAttributes.Normal);+ }++ CsvFormat.WriteRows([["a", "b,c"]], csv);+ Assert.Equal([0xEF, 0xBB, 0xBF, .. Encoding.UTF8.GetBytes("a,\"b,c\"\r\n")], File.ReadAllBytes(csv));+ Assert.Equal([csv], Directory.GetFiles(dir));++ static IEnumerable<IEnumerable<string>> FailingRows()+ {+ yield return ["a", "b"];+ throw new IOException("the second row failed");+ }+ }+[Fact]public async Task Xlsx_to_hwpx_makes_a_table_per_sheet(){