Skip to content

armgit / Filee

Public Imported from GitHub
0

Free, open-source file converter. Convert 263 formats offline by drag & drop — images, PDF, Word, Excel, PowerPoint, HWP/HWPX, video, audio, e-books and archives. No upload, no Office needed.

FIT: summarize sessions while reading, drop non-finite floats, GPX notice only when the start is lost

SeoJCalivef0c3719bbf8a
diff --git a/docs/ENGINES.md b/docs/ENGINES.md
index b89100a9..cd7db0ac 100644
--- a/docs/ENGINES.md
+++ b/docs/ENGINES.md
@@ -1093,8 +1093,8 @@ attribute: the earliest session start, else the first record time (for a single
keeps what it can:
- **GPX**: a track (a waypoint for a single position) with `<time>` and `<ele>` on the points that have them, and the
- sport as `<type>`. A track has no start time of its own: when the start differs from the first point's time, or the
- first point has no time, a notice says the start was left out.
+ sport as `<type>`. A track has no start time of its own, but its points keep their times: only when no point has a
+ time and the activity has a start does a notice say the start was left out.
- **KML / KMZ**: a Point for a single position. A track is a `gx:Track` when every point has a time and a height;
otherwise a LineString, with the point times in a `coordTimes` `Data` entry when any point has one. `type` and `time`
are `ExtendedData`.
diff --git a/src/Filee.Engines/Geo/FitFormat.cs b/src/Filee.Engines/Geo/FitFormat.cs
index ac4eced1..9fe9f3b9 100644
--- a/src/Filee.Engines/Geo/FitFormat.cs
+++ b/src/Filee.Engines/Geo/FitFormat.cs
@@ -11,15 +11,23 @@ namespace Filee.Engines.Geo;
internal sealed record FitRecord(DateTime? Time, double? Latitude, double? Longitude, double? Elevation,
int? HeartRate, int? Cadence, int? Power, double? Speed, double? Distance);
-internal sealed record FitSession(DateTime? StartTime, int? Sport)
-{
- public string? SportName => Sport is { } sport ? FitFormat.SportName(sport) : null;
-}
-
internal sealed class FitActivity
{
public List<FitRecord> Records { get; } = [];
- public List<FitSession> Sessions { get; } = [];
+
+ /// <summary>The earliest session start. Sessions aren't kept one by one, so their number can't grow memory.</summary>
+ public DateTime? SessionStart { get; private set; }
+
+ /// <summary>The session sports in file order, each once (at most 255).</summary>
+ public List<string> Sports { get; } = [];
+
+ internal void AddSession(DateTime? start, int? sport)
+ {
+ if (start is { } time && (SessionStart is null || time < SessionStart))
+ SessionStart = time;
+ if (sport is { } value && FitFormat.SportName(value) is var name && !Sports.Contains(name))
+ Sports.Add(name);
+ }
}
internal static class FitFormat
@@ -75,11 +83,10 @@ internal static class FitFormat
throw new ConversionRefusedException("error.fit_no_positions");
var properties = new List<KeyValuePair<string, object?>>();
- var sports = activity.Sessions.Select(session => session.SportName).OfType<string>().Distinct().ToArray();
- if (sports.Length > 0)
- properties.Add(new("type", string.Join(", ", sports)));
+ if (activity.Sports.Count > 0)
+ properties.Add(new("type", string.Join(", ", activity.Sports)));
DateTime? start = positioned.Count == 1 ? positioned[0].Time
- : activity.Sessions.Select(session => session.StartTime).Where(time => time is not null).Order().FirstOrDefault()
+ : activity.SessionStart
?? activity.Records.Select(record => record.Time).FirstOrDefault(time => time is not null);
if (start is { } time)
properties.Add(new("time", time.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture)));
@@ -218,7 +225,7 @@ internal static class FitFormat
switch (definition.Message)
{
case 0: fileIdSeen = true; break;
- case 18: activity.Sessions.Add(new FitSession(ToDate(startTime), sport)); break;
+ case 18: activity.AddSession(ToDate(startTime), sport); break;
case 20:
if (activity.Records.Count >= maxRecords)
throw new ConversionRefusedException("error.fit_too_large", "2000000 records");
@@ -272,8 +279,12 @@ internal static class FitFormat
_ => bigEndian ? BinaryPrimitives.ReadUInt64BigEndian(bytes) : BinaryPrimitives.ReadUInt64LittleEndian(bytes),
};
if (type is 8 or 9)
- return raw == (width == 4 ? uint.MaxValue : ulong.MaxValue) ? null
- : type == 8 ? BitConverter.Int32BitsToSingle((int)raw) : BitConverter.Int64BitsToDouble((long)raw);
+ {
+ if (raw == (width == 4 ? uint.MaxValue : ulong.MaxValue))
+ return null;
+ double number = type == 8 ? BitConverter.Int32BitsToSingle((int)raw) : BitConverter.Int64BitsToDouble((long)raw);
+ return double.IsFinite(number) ? number : null; // NaN and infinities pass no range check
+ }
if (type is 10 or 11 or 12 or 16 && raw == 0)
return null;
if (type is 1 or 3 or 5 or 14)
diff --git a/src/Filee.Engines/Geo/GeoConverter.cs b/src/Filee.Engines/Geo/GeoConverter.cs
index d1c538f9..c8144de9 100644
--- a/src/Filee.Engines/Geo/GeoConverter.cs
+++ b/src/Filee.Engines/Geo/GeoConverter.cs
@@ -59,10 +59,9 @@ public sealed class GeoConverter(ILogger? logger = null) : IConverter
_logger.LogDebug("{File}: {Features} features read as {Format}, {Crs}", step.InputPath, dataset.Features.Count, step.From, dataset.Crs.Name);
if (dataset.RowsWithoutLocation)
notices.Add("notice.geo_rows_without_location");
- if (activity is not null && step.To == "gpx" && dataset.Features[0].Geometry is GeoLineString line
- && dataset.Features[0].FindText("time") is { } summary
- && (line.Times is null || line.Times[0] is not { } first
- || first.ToString("yyyy-MM-ddTHH:mm:ssZ", System.Globalization.CultureInfo.InvariantCulture) != summary))
+ // A GPX track has no start time, but its points keep theirs: the start is lost only when none has one.
+ if (activity is not null && step.To == "gpx" && dataset.Features[0].Geometry is GeoLineString { Times: null }
+ && dataset.Features[0].FindText("time") is not null)
notices.Add("notice.geo_gpx_lossy");
}
progress?.Report(0.6);
diff --git a/tests/Filee.Engines.Tests/FitFormatTests.cs b/tests/Filee.Engines.Tests/FitFormatTests.cs
index 35ac5138..59e9e16f 100644
--- a/tests/Filee.Engines.Tests/FitFormatTests.cs
+++ b/tests/Filee.Engines.Tests/FitFormatTests.cs
@@ -78,19 +78,20 @@ public class FitFormatTests(EngineFixture fx) : IClassFixture<EngineFixture>
}
[Fact]
- public async Task Gpx_only_reports_a_different_or_unrepresentable_summary_start()
+ public async Task Gpx_reports_the_start_only_when_no_point_keeps_a_time()
{
- var records = Three();
- foreach (var (start, notice) in new (uint?, string?)[]
+ var firstUntimed = Three();
+ firstUntimed[0].Item1 = null;
+ foreach (var (start, records) in new (uint?, (uint?, int?, int?, ushort?)[])[]
{
- (null, null), (First - 10, "notice.geo_gpx_lossy"),
+ (null, Three()), (First - 10, Three()), (First, Three()), (First - 10, firstUntimed),
})
{
var builder = new FitBuilder().FileId();
if (start is { } t) builder.Session(t, 1);
var input = Save(Append(builder, records));
var job = await fx.ConvertAsync(input, "gpx");
- Assert.Equal(notice, job.Files.Single().NoticeKey);
+ Assert.Null(job.Files.Single().NoticeKey);
}
var withoutTimes = Save(Append(new FitBuilder().FileId().Session(First, 1), Three(times: false)));
Assert.Equal("notice.geo_gpx_lossy", (await fx.ConvertAsync(withoutTimes, "gpx")).Files.Single().NoticeKey);
@@ -98,6 +99,40 @@ public class FitFormatTests(EngineFixture fx) : IClassFixture<EngineFixture>
Assert.Equal(Stamp(First), Child(Assert.Single(Elements(point, "wpt")), "time"));
}
+ [Fact]
+ public async Task Non_finite_float_values_are_left_out_like_invalid_ones()
+ {
+ static FitBuilder.FitField F32(byte number, float value) => new(number, 0x88, BitConverter.GetBytes(value));
+ static FitBuilder.FitField F64(byte number, double value) => new(number, 0x89, BitConverter.GetBytes(value));
+ FitBuilder.FitField[][] records =
+ [
+ [FitBuilder.U32(253, First), FitBuilder.S32(0, 0x20000000), FitBuilder.S32(1, 0x40000000), FitBuilder.U16(2, 3000)],
+ [FitBuilder.U32(253, First + 1), F32(0, float.NaN), FitBuilder.S32(1, 0x20000000), F64(2, double.PositiveInfinity)],
+ [FitBuilder.U32(253, First + 2), FitBuilder.S32(0, 0x10000000), F64(1, double.NegativeInfinity)],
+ [FitBuilder.U32(253, First + 3), FitBuilder.S32(0, 0x30000000), FitBuilder.S32(1, 0x30000000), F32(2, float.PositiveInfinity)],
+ ];
+ var builder = new FitBuilder().FileId();
+ foreach (var fields in records)
+ builder.Define(0, 20, fields: fields).Message(0, fields);
+ var input = Save(builder);
+
+ var gpx = (await fx.ConvertOkAsync(input, "gpx")).Single();
+ Assert.DoesNotContain("NaN", File.ReadAllText(gpx));
+ Assert.DoesNotContain("Infinity", File.ReadAllText(gpx));
+ var points = Elements(gpx, "trkpt");
+ Assert.Equal(["45", "67.5"], points.Select(p => (string?)p.Attribute("lat")));
+ Assert.Equal(["90", "67.5"], points.Select(p => (string?)p.Attribute("lon")));
+ Assert.Equal(["100", null], points.Select(p => Child(p, "ele")));
+ Assert.Equal(
+ [
+ "time,longitude,latitude,elevation,heart_rate,cadence,power,speed,distance",
+ $"{Stamp(First)},90,45,100,,,,,",
+ $"{Stamp(First + 1)},,,,,,,,",
+ $"{Stamp(First + 2)},,,,,,,,",
+ $"{Stamp(First + 3)},67.5,67.5,,,,,,",
+ ], File.ReadAllLines((await fx.ConvertOkAsync(input, "csv")).Single()));
+ }
+
[Fact]
public async Task Single_position_is_a_point_in_each_spatial_format()
{
diff --git a/tests/Filee.Engines.Tests/FitReaderTests.cs b/tests/Filee.Engines.Tests/FitReaderTests.cs
index 48412425..439fd7fa 100644
--- a/tests/Filee.Engines.Tests/FitReaderTests.cs
+++ b/tests/Filee.Engines.Tests/FitReaderTests.cs
@@ -50,8 +50,8 @@ public class FitReaderTests(EngineFixture fx) : IClassFixture<EngineFixture>
Assert.Equal(5.678, record.Speed);
Assert.Equal(123.45, record.Distance);
Assert.Equal(Time(0x10000040), activity.Records[1].Time);
- Assert.Equal([Time(0x10000035), Time(0x10000030)], activity.Sessions.Select(s => s.StartTime));
- Assert.Equal(["running", "cycling"], activity.Sessions.Select(s => s.SportName));
+ Assert.Equal(Time(0x10000030), activity.SessionStart);
+ Assert.Equal(["running", "cycling"], activity.Sports);
Assert.Equal("200", FitFormat.SportName(200));
Assert.Equal("canoeing", FitFormat.SportName(88));
@@ -162,6 +162,23 @@ public class FitReaderTests(EngineFixture fx) : IClassFixture<EngineFixture>
Assert.Equal("cut short", error.Detail);
}
+ [Fact]
+ public void Many_empty_sessions_are_summed_up_without_keeping_each_one()
+ {
+ var builder = new FitBuilder().FileId().Session(0x10000035, 1).Define(0, 18);
+ for (var i = 0; i < 500_000; i++)
+ builder.Message(0, []);
+ var path = FitBuilder.Save(fx.NewFolder(), builder.Session(0x10000030, 2).Session(0x10000040, 1).Build());
+ // Warm the parser so the measurement holds only this file: its bytes, and no object per session.
+ _ = Read(new FitBuilder().FileId().Session(0x10000035, 1));
+ var before = GC.GetAllocatedBytesForCurrentThread();
+ var activity = FitFormat.Read(path, CancellationToken.None);
+ var allocated = GC.GetAllocatedBytesForCurrentThread() - before;
+ Assert.True(allocated < new FileInfo(path).Length + 1_000_000, $"{allocated} bytes for a {new FileInfo(path).Length}-byte file");
+ Assert.Equal(Time(0x10000030), activity.SessionStart);
+ Assert.Equal(["running", "cycling"], activity.Sports);
+ }
+
[Fact]
public void Configurable_file_and_record_limits_refuse_only_excess()
{