Skip to content

armgit / Filee

Public Imported from GitHub
0

Free, open-source file converter. Convert 263 formats offline by drag & drop — images, PDF, Word, Excel, PowerPoint, HWP/HWPX, video, audio, e-books and archives. No upload, no Office needed.

FIT: filter values by their profile range, keep only records an output uses

SeoJCalivea99f4f1cf31a
diff --git a/src/Filee.App/Assets/i18n/de.json b/src/Filee.App/Assets/i18n/de.json
index fc742a82..eaed5048 100644
--- a/src/Filee.App/Assets/i18n/de.json
+++ b/src/Filee.App/Assets/i18n/de.json
@@ -107,6 +107,14 @@
"error.geo_csv_no_locations": "Keine Zeile der CSV-Datei enthält eine verwendbare Position",
"error.geo_unsupported_shape": "Dieser Geometrietyp wird nicht unterstützt",
"error.geo_too_large": "Das Shapefile würde die Formatgrenze von 2 GB überschreiten",
+ "error.fit_not_fit": "Dies ist keine Garmin-FIT-Datei",
+ "error.fit_damaged": "Die FIT-Datei ist beschädigt oder unvollständig",
+ "error.fit_chained": "Die Datei enthält mehrere FIT-Dateien hintereinander. Das wird noch nicht unterstützt",
+ "error.fit_too_large": "Die FIT-Datei ist zu groß zum Konvertieren",
+ "error.fit_no_file_id": "Die FIT-Datei gibt nicht an, welche Art von Datei sie ist",
+ "error.fit_not_activity": "Diese FIT-Datei ist keine Aktivität (nur Aktivitäten werden konvertiert)",
+ "error.fit_no_positions": "Die Aktivität enthält keine aufgezeichneten Positionen. Konvertiere sie in CSV, XLSX oder JSON",
+ "error.fit_no_records": "Die Aktivität enthält keine aufgezeichneten Daten",
"notice.pdf_not_smaller": "Die Datei ließ sich nicht verkleinern und wurde unverändert kopiert",
"notice.cad_newer_left_out": "Objekte, die die gewählte AutoCAD-Version nicht unterstützt (Multiführungslinien, Netze …), wurden weggelassen",
"notice.original_kept": "Das Original wurde beibehalten und das Ergebnis als Kopie gespeichert",
@@ -129,6 +137,7 @@
"notice.geo_rows_without_location": "Einige Zeilen enthalten keine verwendbare Position und wurden ohne Geometrie beibehalten",
"notice.geo_field_names": "Feldnamen in Shapefiles dürfen höchstens 10 Zeichen lang sein. Einige wurden gekürzt",
"notice.geo_shapefile_split": "Ein Shapefile enthält nur einen Geometrietyp. Punkte, Linien und Polygone wurden deshalb in getrennten Dateien gespeichert",
+ "notice.geo_heights_dropped": "Einige Punkte haben keine Höhe, und ein Shapefile kann sie nicht einzeln weglassen. Deshalb wurden alle Höhen weggelassen",
"engine.reason.not_installed": "Nicht installiert",
"engine.reason.error": "Fehler bei der Prüfung",
"engine.reason.unsupported_platform": "Auf dieser Plattform nicht verfügbar",
diff --git a/src/Filee.Engines/Geo/FitFormat.cs b/src/Filee.Engines/Geo/FitFormat.cs
index 59057986..c24bfe67 100644
--- a/src/Filee.Engines/Geo/FitFormat.cs
+++ b/src/Filee.Engines/Geo/FitFormat.cs
@@ -195,9 +195,10 @@ internal static class FitFormat
Require(field.Size);
var value = ReadNumber(bytes.AsSpan(position, field.Size), field.Type, definition.BigEndian);
position += field.Size;
- // A field read is taken only in its profile width as an integer: a float or wider type could carry
- // any size of value past the range the profile allows.
- if (value is null || ProfileWidth(definition.Message, field.Number) is { } width && IntegerWidth(field.Type) != width)
+ // A value is read in the base type the file defines for it. Unless it is a whole number within the range
+ // of the field's Profile type (short of that type's invalid value), it is invalid whatever that base type is.
+ if (value is null || ProfileRange(definition.Message, field.Number) is { } range
+ && !(value >= range.Min && value <= range.Max && value == Math.Floor(value.Value)))
continue;
if (field.Number == 253 && value is >= 0x10000000 and < 0xFFFFFFFF)
{
@@ -236,8 +237,8 @@ internal static class FitFormat
lat = lon = null;
var elevation = enhancedHeight ?? height;
var recordSpeed = enhancedSpeed ?? speed;
- // A record without a single value adds nothing to any output, so it is counted but not kept.
- if (time is null && lat is null && elevation is null && heartRate is null && cadence is null
+ // A record with nothing a CSV row or a track uses (a height alone is neither) is counted but not kept.
+ if (time is null && lat is null && heartRate is null && cadence is null
&& power is null && recordSpeed is null && distance is null)
break;
activity.Records.Add(new FitRecord(ToDate(time), lat, lon, elevation,
@@ -260,25 +261,17 @@ internal static class FitFormat
}
}
- /// <summary>The integer width the FIT Profile gives each field read here; other fields aren't width-checked.</summary>
- private static int? ProfileWidth(int message, int number) => (message, number) switch
+ /// <summary>The valid raw values of each field read here: its Profile type's range short of that type's invalid
+ /// value (the largest value for unsigned and enum types, int.MaxValue for the sint32 positions).</summary>
+ private static (double Min, double Max)? ProfileRange(int message, int number) => (message, number) switch
{
- (_, 253) or (18, 2) or (20, 0) or (20, 1) or (20, 5) or (20, 73) or (20, 78) => 4,
- (20, 2) or (20, 6) or (20, 7) => 2,
- (0, 0) or (18, 5) or (20, 3) or (20, 4) => 1,
+ (20, 0) or (20, 1) => (int.MinValue, int.MaxValue - 1),
+ (_, 253) or (18, 2) or (20, 5) or (20, 73) or (20, 78) => (0, uint.MaxValue - 1),
+ (20, 2) or (20, 6) or (20, 7) => (0, ushort.MaxValue - 1),
+ (0, 0) or (18, 5) or (20, 3) or (20, 4) => (0, byte.MaxValue - 1),
_ => null,
};
- /// <summary>The width of an integer base type (enum, signed, unsigned and their invalid-zero kinds); 0 otherwise.</summary>
- private static int IntegerWidth(byte baseType) => (baseType & 0x1F) switch
- {
- 0 or 1 or 2 or 10 => 1,
- 3 or 4 or 11 => 2,
- 5 or 6 or 12 => 4,
- 14 or 15 or 16 => 8,
- _ => 0,
- };
-
private static DateTime? ToDate(uint? seconds) =>
seconds is { } value ? DateTime.UnixEpoch.AddSeconds(value + FitEpochOffset) : null;
@@ -331,19 +324,14 @@ internal static class FitFormat
&& type is not (10 or 11 or 12 or 16) ? null : raw;
}
- /// <summary>Writes one nonempty record per CSV row, keeping file order and empty sensor cells.</summary>
+ /// <summary>Writes one CSV row per kept record, keeping file order and empty sensor cells.</summary>
public static void WriteCsv(FitActivity activity, string path)
{
- if (!activity.Records.Any(HasRow))
+ if (activity.Records.Count == 0)
throw new ConversionRefusedException("error.fit_no_records");
string[] header = ["time", "longitude", "latitude", "elevation", "heart_rate", "cadence", "power", "speed", "distance"];
// Rows are made as they are written, so a long activity never holds its whole CSV as text.
- CsvFormat.WriteRows(activity.Records.Where(HasRow).Select(Row).Prepend(header), path);
-
- static bool HasRow(FitRecord record) =>
- record.Time is not null || record.Latitude is not null || record.Longitude is not null
- || record.HeartRate is not null || record.Cadence is not null || record.Power is not null
- || record.Speed is not null || record.Distance is not null;
+ CsvFormat.WriteRows(activity.Records.Select(Row).Prepend(header), path);
static string[] Row(FitRecord record) =>
[
diff --git a/src/Filee.Engines/Office/Sheets/CsvFormat.cs b/src/Filee.Engines/Office/Sheets/CsvFormat.cs
index 2cc65a62..9221d3e1 100644
--- a/src/Filee.Engines/Office/Sheets/CsvFormat.cs
+++ b/src/Filee.Engines/Office/Sheets/CsvFormat.cs
@@ -78,7 +78,8 @@ internal static class CsvFormat
// Replacing a file needs only the folder's permission on Unix: refuse what writing into the file would refuse.
if (File.Exists(path))
using (new FileStream(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) { }
- var partial = Path.Combine(Path.GetDirectoryName(Path.GetFullPath(path))!, $".{Path.GetFileName(path)}.{Guid.NewGuid():N}.tmp");
+ var partial = Path.Combine(Path.GetDirectoryName(Path.GetFullPath(path))!,
+ $".{Path.GetFileName(path)}.{Guid.NewGuid().ToString("N")[..8]}.filee-tmp");
try
{
using (var writer = new StreamWriter(partial, append: false, new UTF8Encoding(encoderShouldEmitUTF8Identifier: true)))
@@ -93,7 +94,14 @@ internal static class CsvFormat
}
catch
{
- File.Delete(partial);
+ try
+ {
+ File.Delete(partial);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ // The partial file stays behind (a scanner may hold it); the row's own error is what reaches the job.
+ }
throw;
}
}
diff --git a/tests/Filee.Engines.Tests/FitFormatTests.cs b/tests/Filee.Engines.Tests/FitFormatTests.cs
index 1776f36d..8e51df6c 100644
--- a/tests/Filee.Engines.Tests/FitFormatTests.cs
+++ b/tests/Filee.Engines.Tests/FitFormatTests.cs
@@ -2,6 +2,7 @@ using System.Buffers.Binary;
using System.IO.Compression;
using System.Text.Json;
using System.Xml.Linq;
+using Filee.Core.Conversion;
using Filee.Core.Formats;
using Filee.Engines.Geo;
@@ -134,15 +135,29 @@ public class FitFormatTests(EngineFixture fx) : IClassFixture<EngineFixture>
}
[Fact]
- public async Task Fields_not_in_their_profile_integer_width_are_left_out()
+ public async Task Values_past_their_profile_range_are_left_out_whatever_their_base_type()
{
- // Height and heart rate as float32 or a wider integer would carry values far past what the profile allows.
- FitBuilder.FitField[] fields =
+ // A float or wider integer can carry values far past what the profile allows; those are dropped.
+ FitBuilder.FitField[] wild =
[
FitBuilder.U32(253, First), FitBuilder.S32(0, 0x20000000), FitBuilder.S32(1, 0x40000000),
new(2, 0x88, BitConverter.GetBytes(3e38f)), FitBuilder.U32(3, 4_000_000_000), new(4, 0x88, BitConverter.GetBytes(1e30f)),
];
- var input = Save(new FitBuilder().FileId().Define(0, 20, fields: fields).Message(0, fields));
+ // FIT reads a value in the base type its definition gives, so a uint16 heart rate or a uint32 height is valid.
+ // A negative sint8 cadence is past the uint8 range.
+ FitBuilder.FitField[] wide =
+ [
+ FitBuilder.U32(253, First + 1), FitBuilder.U32(2, 3000), FitBuilder.U16(3, 150), new(4, 0x01, [unchecked((byte)-5)]),
+ ];
+ // A fraction is not a valid integer value, nor is the sint32 invalid longitude written in a wider type; a whole
+ // float cadence and a sint32 power are valid.
+ FitBuilder.FitField[] mixed =
+ [
+ FitBuilder.U32(253, First + 2), FitBuilder.S32(0, 0x20000000), new(1, 0x8E, BitConverter.GetBytes((long)int.MaxValue)),
+ new(3, 0x88, BitConverter.GetBytes(72.5f)), new(4, 0x88, BitConverter.GetBytes(90f)), new(7, 0x85, BitConverter.GetBytes(300)),
+ ];
+ var input = Save(new FitBuilder().FileId().Define(0, 20, fields: wild).Message(0, wild)
+ .Define(1, 20, fields: wide).Message(1, wide).Define(2, 20, fields: mixed).Message(2, mixed));
var gpx = (await fx.ConvertOkAsync(input, "gpx")).Single();
Assert.Null(Child(Assert.Single(Elements(gpx, "wpt")), "ele"));
@@ -150,7 +165,16 @@ public class FitFormatTests(EngineFixture fx) : IClassFixture<EngineFixture>
[
"time,longitude,latitude,elevation,heart_rate,cadence,power,speed,distance",
$"{Stamp(First)},90,45,,,,,,",
+ $"{Stamp(First + 1)},,,100,150,,,,",
+ $"{Stamp(First + 2)},,,,,90,300,,",
], File.ReadAllLines((await fx.ConvertOkAsync(input, "csv")).Single()));
+
+ // A file type of 4.5 is not activity (4) cut short: the file says no valid type.
+ FitBuilder.FitField[] halfType = [new(0, 0x88, BitConverter.GetBytes(4.5f))];
+ var refused = Assert.Throws<ConversionRefusedException>(() => FitFormat.Read(
+ Save(new FitBuilder().Define(15, 0, fields: halfType).Message(15, halfType).Define(0, 20, fields: wide).Message(0, wide)),
+ CancellationToken.None));
+ Assert.Equal("error.fit_no_file_id", refused.ErrorKey);
}
[Fact]
diff --git a/tests/Filee.Engines.Tests/FitReaderTests.cs b/tests/Filee.Engines.Tests/FitReaderTests.cs
index 17a58e42..de0b945a 100644
--- a/tests/Filee.Engines.Tests/FitReaderTests.cs
+++ b/tests/Filee.Engines.Tests/FitReaderTests.cs
@@ -183,7 +183,9 @@ public class FitReaderTests(EngineFixture fx) : IClassFixture<EngineFixture>
public void Empty_records_count_toward_the_limit_but_are_not_kept()
{
var time = new[] { FitBuilder.U32(253, 0x1000003B) };
- var builder = new FitBuilder().FileId().Define(1, 20, fields: time).Message(1, time).Define(0, 20);
+ FitBuilder.FitField[] heightOnly = [FitBuilder.U16(2, 2585)];
+ var builder = new FitBuilder().FileId().Define(1, 20, fields: time).Message(1, time)
+ .Define(2, 20, fields: heightOnly).Message(2, heightOnly).Define(0, 20);
for (var i = 0; i < 500_000; i++)
builder.Message(0, []);
var path = FitBuilder.Save(fx.NewFolder(), builder.Build());
@@ -276,10 +278,12 @@ public class FitReaderTests(EngineFixture fx) : IClassFixture<EngineFixture>
FitBuilder.S32(1, 0x40000000), FitBuilder.U16(2, 3000), FitBuilder.U8(3, 72),
FitBuilder.U8(4, 88), FitBuilder.U16(7, 210), FitBuilder.U16(6, 4321),
FitBuilder.U32(5, 12345) };
- var activity = Read(new FitBuilder().FileId().Define(0, 20, fields: fields).Message(0, fields));
- activity.Records.Add(new FitRecord(null, null, null, 17, null, null, null, null, null));
- activity.Records.Add(new FitRecord(null, null, null, null, null, null, null, null, null));
- activity.Records.Add(new FitRecord(null, null, null, 12, 60, null, null, null, null));
+ // A height alone and an empty record are not kept; a height with a heart rate is.
+ FitBuilder.FitField[] heightOnly = [FitBuilder.U16(2, 2585)], heightAndHeart = [FitBuilder.U16(2, 2560), FitBuilder.U8(3, 60)];
+ var activity = Read(new FitBuilder().FileId().Define(0, 20, fields: fields).Message(0, fields)
+ .Define(1, 20, fields: heightOnly).Message(1, heightOnly).Define(2, 20).Message(2, [])
+ .Define(3, 20, fields: heightAndHeart).Message(3, heightAndHeart));
+ Assert.Equal(2, activity.Records.Count);
var path = Path.Combine(fx.NewFolder(), "records.csv");
FitFormat.WriteCsv(activity, path);
var bytes = File.ReadAllBytes(path);
diff --git a/tests/Filee.Engines.Tests/OfficeTests.cs b/tests/Filee.Engines.Tests/OfficeTests.cs
index 564ff3b1..11356304 100644
--- a/tests/Filee.Engines.Tests/OfficeTests.cs
+++ b/tests/Filee.Engines.Tests/OfficeTests.cs
@@ -178,6 +178,41 @@ public class OfficeTests(EngineFixture fx) : IClassFixture<EngineFixture>
}
}
+ [Fact]
+ public void Csv_rows_that_fail_keep_their_error_when_the_partial_file_cannot_be_deleted()
+ {
+ var dir = fx.NewFolder();
+ var csv = Path.Combine(dir, "rows.csv");
+ File.WriteAllText(csv, "kept");
+ FileStream? held = null;
+ try
+ {
+ var error = Assert.Throws<IOException>(() => CsvFormat.WriteRows(Rows(), csv));
+ Assert.Equal("the second row failed", error.Message);
+ Assert.Equal("kept", File.ReadAllText(csv));
+ }
+ finally
+ {
+ held?.Dispose();
+ if (!OperatingSystem.IsWindows())
+ File.SetUnixFileMode(dir, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute);
+ foreach (var partial in Directory.GetFiles(dir, "*.filee-tmp"))
+ File.Delete(partial);
+ }
+
+ IEnumerable<IEnumerable<string>> Rows()
+ {
+ yield return ["a"];
+ // From here deleting the partial file fails: on Windows while a handle without delete sharing is open,
+ // on Unix in a folder that can't be written.
+ if (OperatingSystem.IsWindows())
+ held = new FileStream(Assert.Single(Directory.GetFiles(dir, "*.filee-tmp")), FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+ else
+ File.SetUnixFileMode(dir, UnixFileMode.UserRead | UnixFileMode.UserExecute);
+ throw new IOException("the second row failed");
+ }
+ }
+
[Fact]
public async Task Xlsx_to_hwpx_makes_a_table_per_sheet()
{