Skip to content

Next Gen STEP AP242 Native-ish BREP Geometric Modeling Kernel

27.4 KB
Raw Blame
using System.Text;
using Aetheris.Kernel.Firmament.Scene;
using Aetheris.Kernel.Firmament.Assembly;
using Aetheris.Kernel.Firmament.FirmamentV2.Syntax;

namespace Aetheris.Kernel.Firmament.FirmamentV2;

public sealed record FirmamentLanguageToken(int Start, int Length, string Kind);
public sealed record FirmamentLanguageDiagnostic(string Severity, string Code, string Message, int Start, int Length);
public sealed record FirmamentLanguageAnalysis(string Document, string Revision,
    IReadOnlyList<FirmamentLanguageToken> Tokens, IReadOnlyList<FirmamentLanguageDiagnostic> Diagnostics);
public sealed record FirmamentLanguageLocation(string Document, int Start, int Length);
public sealed record FirmamentLanguageHover(string Document, string Revision, int Start, int Length, string Title, string Description);
public sealed record FirmamentLanguageFormat(string Document, string Revision, string Text, bool Changed);

/// <summary>
/// Source-side projection of the existing Firmament parser and generated semantic schema.
/// It never constructs a BRep. Lexical spans support editor colors and source navigation;
/// parser diagnostics and construct/field descriptions remain compiler owned.
/// </summary>
public static class FirmamentLanguageAnalysisService
{
    internal sealed record Lexeme(int Start, int Length, string Text, string Shape, string? Error = null);
    private static readonly HashSet<string> Keywords = ["Model", "Units", "Modify", "WireForm", "Construction", "Use", "schema", "Let", "Static", "Enum", "Options", "match", "Match", "With", "with", "Expose", "Include", "Subassembly", "Assembly", "Part", "Template", "Feature", "return", "Function", "Record", "Table", "Set", "Assert", "measure", "Pattern", "Over", "Using", "As", "as", "Output", "tol", "Concept", "Require", "Motion", "Track", "Keyframes", "Keyframe"];
    private static readonly HashSet<string> DeclarationKinds = ["Model", "Struct", "Subassembly", "Assembly", "Template", "Profile", "Static", "Record", "Feature", "Function", "Table", "Set", "Pattern", "Section", "SectionChain", "FrameTransform", "Points", "Path", "Semantic", "Enum", "Options", "Concept", "Motion", "Track", "Keyframes", "Keyframe"];
    private static readonly HashSet<string> Types = ["Length", "Angle", "Time", "Float", "Int", "Bool", "Box3", "Plane", "Point2", "Point3"];
    private static readonly HashSet<string> Choices = FirmamentSemanticSchemas.All
        .SelectMany(construct => construct.Fields).SelectMany(field => field.Choices).ToHashSet(StringComparer.Ordinal);

    public static FirmamentLanguageAnalysis Analyze(string source, string document, string revision,
        string? sourceDirectory = null, IReadOnlyDictionary<string, string>? resources = null)
    {
        ArgumentNullException.ThrowIfNull(source);
        var lexemes = Lex(FirmamentSourceSpelling.Normalize(source));
        var syntaxReport = Syntax.FirmamentSyntaxParser.Parse(source,document);
        var documentSyntax = syntaxReport.Document;
        var hasConcreteRoot = documentSyntax.Statements.OfType<Syntax.DeclarationSyntax>().Any(d =>
            d.Type.Name.Text is "Model" or "model" or "Struct" && !d.Modifiers.Any(m => m.Name.Text is "Template" or "Concept"));
        var tokens = lexemes.Select((lexeme, index) => new FirmamentLanguageToken(
            lexeme.Start, lexeme.Length, Classify(lexemes, index))).ToArray();
        if(Structural.StructuralAuthoring.IsStructuralSource(source))
            return new(document,revision,tokens,Structural.StructuralAuthoring.Analyze(source,document).Select(d=>
            {
                var located=FirmamentBuildAdmissionValidation.Present(d,document,source);
                return new FirmamentLanguageDiagnostic(located.Severity,located.Code,located.Message,located.SourceSpan?.Start??0,located.SourceSpan?.Length??0);
            }).ToArray());
        if (Garment.GarmentAuthoring.HasRoot(source))
        {
            var garment = Garment.GarmentCompiler.Compile(source, document);
            return new(document, revision, tokens, garment.Diagnostics.Select(diagnostic =>
                new FirmamentLanguageDiagnostic("error", diagnostic.Code, diagnostic.Message, 0, 0)).ToArray());
        }
        if (SceneAuthoring.HasRoot(source))
        {
            var scene = SceneAuthoring.Parse(source, document);
            return new(document, revision, tokens, scene.Diagnostics.Select(d => new FirmamentLanguageDiagnostic(
                "error", d.Code, d.Message, 0, 0)).ToArray());
        }
        // Imported authoring modules need project context. Do not feed them into
        // the unrelated concrete-part parser and manufacture syntax errors.
        if (lexemes.Any(t => t.Text == "Include" && t.Shape == "identifier")
            && !hasConcreteRoot)
            return new(document, revision, tokens, [new("information", "firmament-language-project-context-required", "Imported declarations require project-aware analysis.", 0, 0)]);
        if (HasAssemblyRoot(source))
        {
            var assembly = new AssemblyM0Parser().Parse(source, document);
            return new(document, revision, tokens, assembly.Diagnostics.Select(d => new FirmamentLanguageDiagnostic(d.Severity == AssemblyDiagnosticSeverity.Error ? "error" : "information", d.Code, d.Message, 0, 0)).ToArray());
        }
        if(Syntax.FirmamentSyntaxTraversal.Declarations(documentSyntax).FirstOrDefault(d=>d.Type.Name.Text is "PlasticShell" or "BodyState" or "SculptState" or "MeshState"
            || d.Type.Name.Text=="LinearElastic" && d.Modifiers.Any(m=>m.Name.Text is "Analysis" or "analysis")) is {} externalDomain)
        {
            // Editor analysis never materializes sculpt states or runs engineering.
            // The owning compilation supplies semantic/capability diagnostics.
            var messages=syntaxReport.Diagnostics.Select(d=>new FirmamentLanguageDiagnostic(d.Severity=="warning"?"warning":"error",d.Code,d.Message,d.SourceSpan?.Start??0,d.SourceSpan?.Length??0)).ToList();
            if(syntaxReport.IsSuccess)messages.Add(new("information","firmament-language-domain-context-required",
                "Shared syntax checked only. Use this domain's build/validate compiler for semantic, capability and geometry diagnostics.",externalDomain.Span.Start,externalDomain.Span.Length));
            return new(document,revision,tokens,messages);
        }
        if (SectionChainAuthoringParser.IsSectionChainSource(source))
        {
            var section = SectionChainAuthoringParser.Compile(source, materialize: false);
            return new(document, revision, tokens, section.Diagnostics.Select(d => LocateDiagnostic(source, lexemes, d)).ToArray());
        }
        if (Materializer.WireRouteAuthoring.IsSource(source) && !lexemes.Any(t => t.Shape == "identifier" && t.Text == "Template"))
        {
            var route = Materializer.WireFormAuthoring.Parse(source);
            return new(document, revision, tokens, route.Diagnostics.Select(d => new FirmamentLanguageDiagnostic("error", d.Source, d.Message, 0, 0)).ToArray());
        }
        if (!hasConcreteRoot && lexemes.Any(t => t.Shape == "identifier" && t.Text is "Template" or "Subassembly" or "Appearance" or "Material"))
            return new(document, revision, tokens, [new("information", "firmament-language-project-context-required", "Declaration modules are validated in their consuming project.", 0, 0)]);
        var parse = FirmamentV2Parser.Parse(source, sourceDirectory, null, null, resources);
        var failures = parse.Diagnostics.Where(item => FirmamentV2Parser.IsFatalDiagnosticCode(item) || parse.SyntaxDiagnostics.Any(d => d.Code == item && d.Severity == "warning") ||
            item == "firmament-v2-parse-failed").Distinct(StringComparer.Ordinal).ToArray();
        if (failures.Length > 1) failures = failures.Where(item => item != "firmament-v2-parse-failed").ToArray();
        var diagnostics = failures.Select(item => parse.SyntaxDiagnostics.FirstOrDefault(d => d.Code == item || item.StartsWith(d.Code + ":", StringComparison.Ordinal)) is { SourceSpan: { } span } located
            ? new FirmamentLanguageDiagnostic(located.Severity == "warning" ? "warning" : "error", located.Code, located.Message, span.Start, span.Length)
            : LocateDiagnostic(source, lexemes, item)).ToArray();
        return new(document, revision, tokens, diagnostics);
    }

    public static FirmamentLanguageHover? Hover(string source, string document, string revision, int offset)
    {
        var lexemes = Lex(FirmamentSourceSpelling.Normalize(source));
        var index = lexemes.FindIndex(item => item.Start <= offset && offset <= item.Start + item.Length && item.Shape == "identifier");
        if (index < 0) return null;
        var token = lexemes[index];
        var construct = FirmamentSemanticSchemas.All.FirstOrDefault(item => item.Name == token.Text);
        if (construct is not null)
            return new(document, revision, token.Start, token.Length, construct.Name, construct.Description ?? construct.Context ?? "Firmament construct");
        var field = FirmamentLanguageService.FieldAt(FirmamentSourceSpelling.Normalize(source), token.Start, token.Text);
        if (field is not null && Classify(lexemes, index) == "field")
            return new(document, revision, token.Start, token.Length, field.Name,
                $"{field.Type}{(field.Required ? " · required" : "")}. {field.Meaning}".Trim());
        var definition = Definition(source, document, offset);
        if (definition is not null)
        {
            var declaration = FirmamentSyntaxTraversal.Declarations(FirmamentSyntaxParser.Parse(source).Document)
                .FirstOrDefault(d => d.Name?.Span.Start == definition.Start);
            if (declaration is not null)
                return new(document, revision, token.Start, token.Length, token.Text,
                    "```firmament\n" + source.Substring(declaration.Span.Start, (declaration.Body?.Span.Start ?? declaration.Span.Start + declaration.Span.Length) - declaration.Span.Start).Trim() + "\n```");
            var parameter = FirmamentSyntaxTraversal.Declarations(FirmamentSyntaxParser.Parse(source).Document)
                .SelectMany(d => d.Parameters).FirstOrDefault(p => p.Name?.Span.Start == definition.Start);
            if (parameter is not null)
                return new(document, revision, token.Start, token.Length, token.Text,
                    "Parameter: `" + source.Substring(parameter.Span.Start, parameter.Span.Length) + "`");
        }
        return null;
    }

    public static FirmamentLanguageLocation? Definition(string source, string document, int offset)
    {
        var tokens = Lex(FirmamentSourceSpelling.Normalize(source));
        var selected = tokens.FirstOrDefault(item => item.Shape == "identifier" && item.Start <= offset && offset <= item.Start + item.Length);
        if (selected is null) return null;
        var syntax = FirmamentSyntaxParser.Parse(source).Document;
        if (FirmamentSyntaxTraversal.Expressions(syntax).OfType<ReferenceSyntax>().Any(r => r.Span.Start == selected.Start))
        {
            var parameters = FirmamentSyntaxTraversal.Declarations(syntax).Where(d => d.Body is { } body &&
                body.ContentSpan.Start <= offset && offset <= body.ContentSpan.Start + body.ContentSpan.Length)
                .SelectMany(d => d.Parameters).Where(p => p.Name?.Text == selected.Text).ToArray();
            if (parameters.Length == 1) return new(document, parameters[0].Name!.Span.Start, parameters[0].Name!.Span.Length);
        }
        var declarations = Declarations(tokens, selected.Text, ReferenceKind(tokens, selected)).ToArray();
        return declarations.Length == 1 ? new(document, declarations[0].Start, declarations[0].Length) : null;
    }

    /// <summary>Lexical source navigation only; never grants access to a private placement port.</summary>
    public static FirmamentLanguageLocation? Definition(FirmamentProjectSnapshot project, string document, int offset)
    {
        if (!project.TryResolve(document, out var source)) return null;
        var local = Definition(source, document, offset);
        if (local is not null) return local;
        var tokens = Lex(FirmamentSourceSpelling.Normalize(source));
        var selected = tokens.FirstOrDefault(t => t.Shape == "identifier" && t.Start <= offset && offset < t.Start + t.Length);
        if (selected is null) return null;
        var matches = project.SourceDocuments.SelectMany(d => Declarations(Lex(d.Value), selected.Text, ReferenceKind(tokens, selected))
            .Select(t => new FirmamentLanguageLocation(d.Key, t.Start, t.Length))).ToArray();
        return matches.Length == 1 ? matches[0] : null;
    }

    public static FirmamentLanguageAnalysis Analyze(FirmamentProjectSnapshot project, string document, string revision)
    {
        if (!project.TryResolve(document, out var source)) throw new ArgumentException("Document is absent from the snapshot.", nameof(document));
        if (!project.TryResolve(project.RootDocument, out var rootSource)
            || !HasAssemblyRoot(rootSource, project.RootDocument, project))
            return Analyze(source, document, revision, Path.GetDirectoryName(document), project.Documents);
        if (SceneAuthoring.HasRoot(source) || Garment.GarmentAuthoring.HasRoot(source)) return Analyze(source,document,revision);
        var tokens = Lex(FirmamentSourceSpelling.Normalize(source));
        var parsed = new AssemblyM0Parser().ParseProject(project);
        return new(document, revision, tokens.Select((t, i) => new FirmamentLanguageToken(t.Start, t.Length, Classify(tokens, i))).ToArray(),
            parsed.Diagnostics.Select(d => new FirmamentLanguageDiagnostic(d.Severity == AssemblyDiagnosticSeverity.Error ? "error" : "information", d.Code, d.Message, 0, 0)).ToArray());
    }

    public static bool HasAssemblyRoot(string source) => HasAssemblyRoot(source, null);

    public static bool HasAssemblyRoot(string source, string? document, FirmamentProjectSnapshot? project = null)
    {
        var tokens = Lex(source).Where(t => t.Shape != "comment").ToArray();
        var depth = 0;
        for (var i = 0; i + 1 < tokens.Length; i++)
        {
            if (tokens[i].Text == "{") depth++;
            else if (tokens[i].Text == "}") depth--;
            if (depth == 0 && tokens[i].Text == "<" && tokens[i + 1].Text == "Assembly") return true;
            if (depth == 0 && tokens[i].Text == "Assembly" && tokens[i + 1].Shape == "identifier" && (i == 0 || tokens[i - 1].Text != "/")) return true;
        }
        if (document is not null && Aetheris.Kernel.Firmament.Modules.FirmamentModuleGraph.HasImports(source))
        {
            var graph = Aetheris.Kernel.Firmament.Modules.FirmamentModuleGraph.Load(source, document, project);
            if (graph.Diagnostics.Count == 0)
                return HasAssemblyRoot(graph.ProjectSource(graph.Root).Text);
        }
        return false;
    }

    private static string? ReferenceKind(IReadOnlyList<Lexeme> tokens, Lexeme selected)
    {
        var at = tokens.ToList().FindIndex(t => t.Start == selected.Start);
        return at >= 2 && tokens[at - 1].Text == ":" && tokens[at - 2].Text is "Material" or "Appearance" or "Profile"
            ? tokens[at - 2].Text : null;
    }

    private static IEnumerable<Lexeme> Declarations(IReadOnlyList<Lexeme> tokens, string name, string? kind = null)
    {
        for (var i = 1; i + 1 < tokens.Count; i++)
        {
            if (tokens[i].Shape != "identifier" || tokens[i].Text != name) continue;
            var previous = i - 1;
            if (tokens[previous].Text == ">")
            {
                while (previous >= 0 && tokens[previous].Text != "<") previous--;
                previous--;
            }
            if (previous < 0 || !(DeclarationKinds.Contains(tokens[previous].Text) || FirmamentSemanticSchemas.All.Any(s => s.Name == tokens[previous].Text))) continue;
            if (kind is not null && tokens[previous].Text != kind) continue;
            if (tokens[i + 1].Text is "{" or "=" or ":" or "(" or "<") yield return tokens[i];
        }
    }

    private static bool SameFittingGuides(Garment.CompiledGarment before, Garment.CompiledGarment after)
    {
        if (before.FittingGuides.Length != after.FittingGuides.Length)
        {
            return false;
        }
        for (int index = 0; index < before.FittingGuides.Length; index++)
        {
            var original = before.FittingGuides[index];
            var formatted = after.FittingGuides[index];
            if (original.ConceptPlane != formatted.ConceptPlane || original.ReleaseAfter != formatted.ReleaseAfter
                || original.Constraint.Id != formatted.Constraint.Id || original.Constraint.Normal != formatted.Constraint.Normal
                || original.Constraint.Offset != formatted.Constraint.Offset
                || original.Constraint.Compliance != formatted.Constraint.Compliance
                || !original.Constraint.Vertices.SequenceEqual(formatted.Constraint.Vertices))
            {
                return false;
            }
        }
        return true;
    }

    public static FirmamentLanguageFormat Format(string source, string document, string revision)
    {
        if (Garment.GarmentAuthoring.HasRoot(source))
        {
            var before = Garment.GarmentCompiler.Compile(source, document);
            if (!before.IsSuccess)
            {
                throw new InvalidOperationException("Format Document requires valid Garment source. Fix syntax errors first.");
            }
            var garmentFormat = FormatConventions(source, document, revision);
            var after = Garment.GarmentCompiler.Compile(garmentFormat.Text, document);
            if (!after.IsSuccess || before.Garment!.Cloth.ContentKey != after.Garment!.Cloth.ContentKey
                || !SameFittingGuides(before.Garment, after.Garment))
            {
                throw new InvalidOperationException("Garment formatting changed compilation.");
            }
            return garmentFormat;
        }
        if (SceneAuthoring.HasRoot(source))
        {
            if (!SceneAuthoring.Parse(source,document).IsSuccess)
                throw new InvalidOperationException("Format Document requires valid Scene source. Fix syntax errors first.");
            // Preserve generic specialization text, authored ports and pattern keys.
            var scene = FormatConventions(source,document,revision);
            if (!SceneAuthoring.Parse(scene.Text,document).IsSuccess)
                throw new InvalidOperationException("Scene formatting changed parser admission.");
            return scene;
        }
        var sharedSyntax = FirmamentSyntaxParser.Parse(source, document);
        if (sharedSyntax.IsSuccess && (HasAssemblyRoot(source) || !sharedSyntax.Document.Statements.OfType<DeclarationSyntax>().Any(d =>
            d.Type.Name.Text is "Model" or "model" or "Struct" && !d.Modifiers.Any(m => m.Name.Text is "Template" or "Concept"))))
            return FormatConventions(source, document, revision);
        if (!FirmamentV2Parser.Parse(source).IsSuccess)
            throw new InvalidOperationException("Format Document requires valid Firmament source. Fix syntax errors first.");
        var preferred = FirmamentSourceSpelling.Prefer(source);
        var lexemes = Lex(preferred);
        var output = new StringBuilder(source.Length + 64);
        var indent = 0;
        var lineStart = true;
        Lexeme? previous = null;
        foreach (var token in lexemes)
        {
            var gap = previous is null ? string.Empty : preferred[(previous.Start + previous.Length)..token.Start];
            if (token.Text == "}")
            {
                indent = Math.Max(0, indent - 1);
                BreakLine();
            }
            else if (gap.Contains('\n') && token.Text != ";" && previous?.Text != "{") BreakLine();

            if (lineStart) { output.Append(' ', indent * 2); lineStart = false; }
            else if (!(gap.Length == 0 && (previous?.Shape == "number" && token.Shape == "identifier"
                || previous?.Shape == "identifier" && token.Text == "<")) && NeedsSpace(previous?.Text, token.Text)) output.Append(' ');
            output.Append(token.Text);
            if (token.Shape == "comment" || token.Text is "{" or ";" or "}")
            {
                if (token.Text == "{") indent++;
                BreakLine();
            }
            previous = token;
        }
        var formatted = output.ToString().TrimEnd() + "\n";
        if (!Lex(FirmamentSourceSpelling.Normalize(source)).Select(t => (t.Text, t.Shape))
            .SequenceEqual(Lex(FirmamentSourceSpelling.Normalize(formatted)).Select(t => (t.Text, t.Shape))))
            throw new InvalidOperationException("Canonical layout was refused because it changed source tokens.");
        var formattedParse = FirmamentV2Parser.Parse(formatted);
        if (!formattedParse.IsSuccess)
            throw new InvalidOperationException("Canonical layout was refused because it changed parser admission: " + string.Join(", ",
                formattedParse.SyntaxDiagnostics.Select(d => d.Code + ": " + d.Message).Concat(formattedParse.Diagnostics).Distinct()));
        return new(document, revision, formatted, formatted != source);

        void BreakLine()
        {
            if (!lineStart) { output.Append('\n'); lineStart = true; }
        }
    }

    /// <summary>
    /// Conservative project/module pass: only admitted vocabulary spellings change.
    /// Layout, literals, comments, and authored identities remain byte-for-byte intact.
    /// This is not standalone or project validation; use the consuming compiler for that.
    /// </summary>
    public static FirmamentLanguageFormat FormatConventions(string source, string document, string revision)
    {
        var text = FirmamentSourceSpelling.Prefer(source);
        if (FirmamentSourceSpelling.Normalize(text) != FirmamentSourceSpelling.Normalize(source))
            throw new InvalidOperationException("Convention formatting changed canonical source.");
        return new(document, revision, text, text != source);
    }

    private static bool NeedsSpace(string? previous, string current)
    {
        if (previous is null || previous is "(" or "[" or "<" or "." or "+" or "-") return false;
        if (current is ")" or "]" or ">" or "." or ":" or "," or ";" or "(" or "[") return false;
        return true;
    }

    private static string Classify(IReadOnlyList<Lexeme> tokens, int index)
    {
        var token = tokens[index];
        if (token.Shape != "identifier") return token.Shape;
        if (Keywords.Contains(token.Text)) return "keyword";
        if (FirmamentSemanticSchemas.All.Any(item => item.Name == token.Text)) return "construct";
        if (Types.Contains(token.Text)) return "type";
        if (index + 1 < tokens.Count && tokens[index + 1].Text == ":" &&
            FirmamentSemanticSchemas.All.Any(item => item.Fields.Any(field => field.Name == token.Text))) return "field";
        if (token.Text is "mm" or "cm" or "m" or "deg" or "rad" or "s" or "ms") return "unit";
        if (Choices.Contains(token.Text)) return "value";
        if (token.Text == "face" && index + 1 < tokens.Count && tokens[index + 1].Text == "(") return "selector";
        return "identifier";
    }

    private static FirmamentLanguageDiagnostic LocateDiagnostic(string source, IReadOnlyList<Lexeme> tokens, string diagnostic)
    {
        var presented = FirmamentBuildAdmissionValidation.Locate(
            new(diagnostic, "fatal", FirmamentInterop.FirmamentV2DiagnosticAdapter.MessageFor(diagnostic)), source, "inline");
        var message = presented.LocationNote is null ? presented.Message : presented.Message + " " + presented.LocationNote;
        return new("error", diagnostic, message, presented.SourceSpan?.Start ?? 0, presented.SourceSpan?.Length ?? 0);
    }

    internal static List<Lexeme> Lex(string source, bool lossless = false)
    {
        var result = new List<Lexeme>();
        for (var at = 0; at < source.Length;)
        {
            var start = at;
            string shape;
            string? error = null;
            if (char.IsWhiteSpace(source[at]))
            {
                while (at < source.Length && char.IsWhiteSpace(source[at])) at++;
                if (!lossless) continue;
                shape = "whitespace";
            }
            else if (at + 1 < source.Length && source[at] == '/' && source[at + 1] == '/')
            {
                at += 2;
                while (at < source.Length && source[at] != '\n') at++;
                shape = "comment";
            }
            else if (at + 1 < source.Length && source[at] == '/' && source[at + 1] == '*')
            {
                var end = source.IndexOf("*/", at + 2, StringComparison.Ordinal);
                at = end < 0 ? source.Length : end + 2;
                shape = "comment";
                if (end < 0) error = "Unterminated block comment; expected '*/'.";
            }
            else if (source[at] == '"')
            {
                at++;
                var closed = false;
                while (at < source.Length)
                {
                    if (source[at] == '\\' && at + 1 < source.Length) { at += 2; continue; }
                    if (source[at++] != '"') continue;
                    closed = true;
                    break;
                }
                shape = "string";
                if (!closed) error = "Unterminated string literal; expected a closing quote.";
            }
            else if (char.IsLetter(source[at]) || source[at] == '_')
            {
                at++;
                while (at < source.Length && (char.IsLetterOrDigit(source[at]) || source[at] == '_')) at++;
                shape = "identifier";
            }
            else if (source[at] == '.' && result.Count > 0 && result[^1].Shape == "identifier"
                && result[^1].Start + result[^1].Length == at)
            {
                at++;
                shape = "punctuation";
            }
            else if (char.IsDigit(source[at]) && result.Count > 1 && result[^1].Text == "."
                && result[^2].Shape == "identifier" && result[^1].Start + 1 == at)
            {
                // External domain symbols such as StainlessSteel.304_Annealed.
                at++;
                while (at < source.Length && (char.IsLetterOrDigit(source[at]) || source[at] == '_')) at++;
                shape = "identifier";
            }
            else if (char.IsDigit(source[at]) || source[at] == '.' && at + 1 < source.Length && char.IsDigit(source[at + 1]))
            {
                at++;
                while (at < source.Length && char.IsDigit(source[at])) at++;
                if (source[start] != '.' && at + 1 < source.Length && source[at] == '.' && char.IsDigit(source[at + 1]))
                {
                    at++;
                    while (at < source.Length && char.IsDigit(source[at])) at++;
                }
                if (at < source.Length && source[at] is 'e' or 'E')
                {
                    var exponent = at + 1;
                    if (exponent < source.Length && source[exponent] is '+' or '-') exponent++;
                    if (exponent < source.Length && char.IsDigit(source[exponent]))
                    {
                        at = exponent + 1;
                        while (at < source.Length && char.IsDigit(source[at])) at++;
                    }
                }
                shape = "number";
            }
            else { at++; shape = "punctuation"; }
            result.Add(new(start, at - start, source[start..at], shape, error));
        }
        if (lossless) result.Add(new(source.Length, 0, "", "end"));
        return result;
    }
}