Skip to content

armgit / Filee

Public Imported from GitHub
0

Free, open-source file converter. Convert 263 formats offline by drag & drop — images, PDF, Word, Excel, PowerPoint, HWP/HWPX, video, audio, e-books and archives. No upload, no Office needed.

FIT: keep memory near the file size and read fields only in their profile width

SeoJCalive4e03c7fef35c
diff --git a/src/Filee.Engines/Geo/FitFormat.cs b/src/Filee.Engines/Geo/FitFormat.cs
index 9fe9f3b9..59057986 100644
--- a/src/Filee.Engines/Geo/FitFormat.cs
+++ b/src/Filee.Engines/Geo/FitFormat.cs
@@ -136,6 +136,7 @@ internal static class FitFormat
uint? lastTime = null;
int? fileType = null;
var fileIdSeen = false;
+ var recordCount = 0;
while (position < limit)
{
ct.ThrowIfCancellationRequested();
@@ -194,7 +195,9 @@ internal static class FitFormat
Require(field.Size);
var value = ReadNumber(bytes.AsSpan(position, field.Size), field.Type, definition.BigEndian);
position += field.Size;
- if (value is null)
+ // A field read is taken only in its profile width as an integer: a float or wider type could carry
+ // any size of value past the range the profile allows.
+ if (value is null || ProfileWidth(definition.Message, field.Number) is { } width && IntegerWidth(field.Type) != width)
continue;
if (field.Number == 253 && value is >= 0x10000000 and < 0xFFFFFFFF)
{
@@ -227,12 +230,18 @@ internal static class FitFormat
case 0: fileIdSeen = true; break;
case 18: activity.AddSession(ToDate(startTime), sport); break;
case 20:
- if (activity.Records.Count >= maxRecords)
+ if (recordCount++ >= maxRecords)
throw new ConversionRefusedException("error.fit_too_large", "2000000 records");
if (lat is < -90 or > 90 || lon is < -180 or > 180 || lat is null || lon is null)
lat = lon = null;
- activity.Records.Add(new FitRecord(ToDate(time), lat, lon, enhancedHeight ?? height,
- heartRate, cadence, power, enhancedSpeed ?? speed, distance));
+ var elevation = enhancedHeight ?? height;
+ var recordSpeed = enhancedSpeed ?? speed;
+ // A record without a single value adds nothing to any output, so it is counted but not kept.
+ if (time is null && lat is null && elevation is null && heartRate is null && cadence is null
+ && power is null && recordSpeed is null && distance is null)
+ break;
+ activity.Records.Add(new FitRecord(ToDate(time), lat, lon, elevation,
+ heartRate, cadence, power, recordSpeed, distance));
break;
}
}
@@ -251,6 +260,25 @@ internal static class FitFormat
}
}
+ /// <summary>The integer width the FIT Profile gives each field read here; other fields aren't width-checked.</summary>
+ private static int? ProfileWidth(int message, int number) => (message, number) switch
+ {
+ (_, 253) or (18, 2) or (20, 0) or (20, 1) or (20, 5) or (20, 73) or (20, 78) => 4,
+ (20, 2) or (20, 6) or (20, 7) => 2,
+ (0, 0) or (18, 5) or (20, 3) or (20, 4) => 1,
+ _ => null,
+ };
+
+ /// <summary>The width of an integer base type (enum, signed, unsigned and their invalid-zero kinds); 0 otherwise.</summary>
+ private static int IntegerWidth(byte baseType) => (baseType & 0x1F) switch
+ {
+ 0 or 1 or 2 or 10 => 1,
+ 3 or 4 or 11 => 2,
+ 5 or 6 or 12 => 4,
+ 14 or 15 or 16 => 8,
+ _ => 0,
+ };
+
private static DateTime? ToDate(uint? seconds) =>
seconds is { } value ? DateTime.UnixEpoch.AddSeconds(value + FitEpochOffset) : null;
@@ -306,17 +334,19 @@ internal static class FitFormat
/// <summary>Writes one nonempty record per CSV row, keeping file order and empty sensor cells.</summary>
public static void WriteCsv(FitActivity activity, string path)
{
- var rows = new List<List<string>>
- {
- new() { "time", "longitude", "latitude", "elevation", "heart_rate", "cadence", "power", "speed", "distance" },
- };
- foreach (var record in activity.Records)
- {
- if (record.Time is null && record.Latitude is null && record.Longitude is null
- && record.HeartRate is null && record.Cadence is null && record.Power is null
- && record.Speed is null && record.Distance is null)
- continue;
- rows.Add([
+ if (!activity.Records.Any(HasRow))
+ throw new ConversionRefusedException("error.fit_no_records");
+ string[] header = ["time", "longitude", "latitude", "elevation", "heart_rate", "cadence", "power", "speed", "distance"];
+ // Rows are made as they are written, so a long activity never holds its whole CSV as text.
+ CsvFormat.WriteRows(activity.Records.Where(HasRow).Select(Row).Prepend(header), path);
+
+ static bool HasRow(FitRecord record) =>
+ record.Time is not null || record.Latitude is not null || record.Longitude is not null
+ || record.HeartRate is not null || record.Cadence is not null || record.Power is not null
+ || record.Speed is not null || record.Distance is not null;
+
+ static string[] Row(FitRecord record) =>
+ [
record.Time?.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture) ?? "",
record.Longitude is { } lon ? GeoText.Degrees(lon) : "",
record.Latitude is { } lat ? GeoText.Degrees(lat) : "",
@@ -326,11 +356,7 @@ internal static class FitFormat
record.Power?.ToString(CultureInfo.InvariantCulture) ?? "",
record.Speed is { } speed ? GeoText.Number(speed) : "",
record.Distance is { } distance ? GeoText.Number(distance) : "",
- ]);
- }
- if (rows.Count == 1)
- throw new ConversionRefusedException("error.fit_no_records");
- CsvFormat.WriteRows(rows, path);
+ ];
}
private static ConversionRefusedException Damaged(string detail) => new("error.fit_damaged", detail);
diff --git a/src/Filee.Engines/Office/Sheets/CsvFormat.cs b/src/Filee.Engines/Office/Sheets/CsvFormat.cs
index 4b4fc700..2cc65a62 100644
--- a/src/Filee.Engines/Office/Sheets/CsvFormat.cs
+++ b/src/Filee.Engines/Office/Sheets/CsvFormat.cs
@@ -70,13 +70,32 @@ internal static class CsvFormat
File.WriteAllText(path, sb.ToString(), new UTF8Encoding(encoderShouldEmitUTF8Identifier: true));
}
- /// <summary>Writes rows of text as CSV the same way as <see cref="Write"/> (UTF-8 with BOM, CRLF).</summary>
+ /// <summary>Writes rows of text as CSV the same way as <see cref="Write"/> (UTF-8 with BOM, CRLF), one row at a
+ /// time, so a lazily made sequence is never held as text. The rows go to a file beside the target, which replaces
+ /// it only once every row is written: a row that fails to come leaves a file already at the path as it was.</summary>
public static void WriteRows(IEnumerable<IEnumerable<string>> rows, string path, char delimiter = ',')
{
- var sb = new StringBuilder();
- foreach (var row in rows)
- sb.Append(string.Join(delimiter, row.Select(value => Quote(value, delimiter)))).Append("\r\n");
- File.WriteAllText(path, sb.ToString(), new UTF8Encoding(encoderShouldEmitUTF8Identifier: true));
+ // Replacing a file needs only the folder's permission on Unix: refuse what writing into the file would refuse.
+ if (File.Exists(path))
+ using (new FileStream(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) { }
+ var partial = Path.Combine(Path.GetDirectoryName(Path.GetFullPath(path))!, $".{Path.GetFileName(path)}.{Guid.NewGuid():N}.tmp");
+ try
+ {
+ using (var writer = new StreamWriter(partial, append: false, new UTF8Encoding(encoderShouldEmitUTF8Identifier: true)))
+ {
+ foreach (var row in rows)
+ {
+ writer.Write(string.Join(delimiter, row.Select(value => Quote(value, delimiter))));
+ writer.Write("\r\n");
+ }
+ }
+ File.Move(partial, path, overwrite: true);
+ }
+ catch
+ {
+ File.Delete(partial);
+ throw;
+ }
}
internal static List<List<string>> Parse(string text, char delimiter)
diff --git a/tests/Filee.Engines.Tests/FitFormatTests.cs b/tests/Filee.Engines.Tests/FitFormatTests.cs
index 59e9e16f..1776f36d 100644
--- a/tests/Filee.Engines.Tests/FitFormatTests.cs
+++ b/tests/Filee.Engines.Tests/FitFormatTests.cs
@@ -133,6 +133,26 @@ public class FitFormatTests(EngineFixture fx) : IClassFixture<EngineFixture>
], File.ReadAllLines((await fx.ConvertOkAsync(input, "csv")).Single()));
}
+ [Fact]
+ public async Task Fields_not_in_their_profile_integer_width_are_left_out()
+ {
+ // Height and heart rate as float32 or a wider integer would carry values far past what the profile allows.
+ FitBuilder.FitField[] fields =
+ [
+ FitBuilder.U32(253, First), FitBuilder.S32(0, 0x20000000), FitBuilder.S32(1, 0x40000000),
+ new(2, 0x88, BitConverter.GetBytes(3e38f)), FitBuilder.U32(3, 4_000_000_000), new(4, 0x88, BitConverter.GetBytes(1e30f)),
+ ];
+ var input = Save(new FitBuilder().FileId().Define(0, 20, fields: fields).Message(0, fields));
+
+ var gpx = (await fx.ConvertOkAsync(input, "gpx")).Single();
+ Assert.Null(Child(Assert.Single(Elements(gpx, "wpt")), "ele"));
+ Assert.Equal(
+ [
+ "time,longitude,latitude,elevation,heart_rate,cadence,power,speed,distance",
+ $"{Stamp(First)},90,45,,,,,,",
+ ], File.ReadAllLines((await fx.ConvertOkAsync(input, "csv")).Single()));
+ }
+
[Fact]
public async Task Single_position_is_a_point_in_each_spatial_format()
{
diff --git a/tests/Filee.Engines.Tests/FitReaderTests.cs b/tests/Filee.Engines.Tests/FitReaderTests.cs
index 439fd7fa..17a58e42 100644
--- a/tests/Filee.Engines.Tests/FitReaderTests.cs
+++ b/tests/Filee.Engines.Tests/FitReaderTests.cs
@@ -179,6 +179,25 @@ public class FitReaderTests(EngineFixture fx) : IClassFixture<EngineFixture>
Assert.Equal(["running", "cycling"], activity.Sports);
}
+ [Fact]
+ public void Empty_records_count_toward_the_limit_but_are_not_kept()
+ {
+ var time = new[] { FitBuilder.U32(253, 0x1000003B) };
+ var builder = new FitBuilder().FileId().Define(1, 20, fields: time).Message(1, time).Define(0, 20);
+ for (var i = 0; i < 500_000; i++)
+ builder.Message(0, []);
+ var path = FitBuilder.Save(fx.NewFolder(), builder.Build());
+ // Warm the parser so the measurement holds only this file: its bytes, and no object per empty record.
+ _ = Read(new FitBuilder().FileId().Define(1, 20, fields: time).Message(1, time));
+ var before = GC.GetAllocatedBytesForCurrentThread();
+ var activity = FitFormat.Read(path, CancellationToken.None);
+ var allocated = GC.GetAllocatedBytesForCurrentThread() - before;
+ Assert.True(allocated < new FileInfo(path).Length + 1_000_000, $"{allocated} bytes for a {new FileInfo(path).Length}-byte file");
+ Assert.Equal(Time(0x1000003B), Assert.Single(activity.Records).Time);
+ var error = Assert.Throws<ConversionRefusedException>(() => FitFormat.Read(path, CancellationToken.None, maxRecords: 500_000));
+ Assert.Equal("error.fit_too_large", error.ErrorKey);
+ }
+
[Fact]
public void Configurable_file_and_record_limits_refuse_only_excess()
{
@@ -246,8 +265,8 @@ public class FitReaderTests(EngineFixture fx) : IClassFixture<EngineFixture>
.Define(0, 20, fields: fields).Message(0, fields)
.Define(1, 20, fields: empty).Compressed(1, 0, empty).Compressed(1, 1, empty)
.Message(0, absolute).Message(0, fields).Compressed(1, 0x1D, empty));
- Assert.Equal([null, null, null, Time(0x1000003B), null, Time(0x1000003D)],
- activity.Records.Select(r => r.Time));
+ // Records left without any value aren't kept, so a relative timestamp that became absolute would show up here.
+ Assert.Equal([Time(0x1000003B), Time(0x1000003D)], activity.Records.Select(r => r.Time));
}
[Fact]
diff --git a/tests/Filee.Engines.Tests/OfficeTests.cs b/tests/Filee.Engines.Tests/OfficeTests.cs
index 5ff535cb..564ff3b1 100644
--- a/tests/Filee.Engines.Tests/OfficeTests.cs
+++ b/tests/Filee.Engines.Tests/OfficeTests.cs
@@ -144,6 +144,40 @@ public class OfficeTests(EngineFixture fx) : IClassFixture<EngineFixture>
Assert.Equal("90", sheet.Get(1, 1)!.Text);
}
+ [Fact]
+ public void Csv_rows_that_fail_part_way_or_meet_a_read_only_file_leave_it_as_it_was()
+ {
+ var dir = fx.NewFolder();
+ var csv = Path.Combine(dir, "rows.csv");
+ File.WriteAllText(csv, "kept");
+
+ Assert.Throws<IOException>(() => CsvFormat.WriteRows(FailingRows(), csv));
+ Assert.Equal("kept", File.ReadAllText(csv));
+ Assert.Equal([csv], Directory.GetFiles(dir));
+
+ File.SetAttributes(csv, FileAttributes.ReadOnly);
+ try
+ {
+ Assert.Throws<UnauthorizedAccessException>(() => CsvFormat.WriteRows([["a"]], csv));
+ Assert.Equal("kept", File.ReadAllText(csv));
+ Assert.Equal([csv], Directory.GetFiles(dir));
+ }
+ finally
+ {
+ File.SetAttributes(csv, FileAttributes.Normal);
+ }
+
+ CsvFormat.WriteRows([["a", "b,c"]], csv);
+ Assert.Equal([0xEF, 0xBB, 0xBF, .. Encoding.UTF8.GetBytes("a,\"b,c\"\r\n")], File.ReadAllBytes(csv));
+ Assert.Equal([csv], Directory.GetFiles(dir));
+
+ static IEnumerable<IEnumerable<string>> FailingRows()
+ {
+ yield return ["a", "b"];
+ throw new IOException("the second row failed");
+ }
+ }
+
[Fact]
public async Task Xlsx_to_hwpx_makes_a_table_per_sheet()
{