Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
// See www.openfst.org for extensive documentation on this weighted
// finite-state transducer library.
//
// FST abstract base class definition, state and arc iterator interface, and
// suggested base implementation.
#ifndef FST_FST_H_
#define FST_FST_H_
#include <sys/types.h>
#include <cmath>
#include <cstddef>
#include <iostream>
#include <memory>
#include <sstream>
#include <string>
#include <utility>
#include <fst/compat.h>
#include <fst/flags.h>
#include <fst/log.h>
#include <fstream>
#include <fst/arc.h>
#include <fst/memory.h>
#include <fst/properties.h>
#include <fst/register.h>
#include <fst/symbol-table.h>
#include <fst/util.h>
DECLARE_bool(fst_align);
namespace fst {
bool IsFstHeader(std::istream &, const string &);
class FstHeader;
template <class Arc>
struct StateIteratorData;
template <class Arc>
struct ArcIteratorData;
template <class Arc>
class MatcherBase;
struct FstReadOptions {
// FileReadMode(s) are advisory, there are many conditions than prevent a
// file from being mapped, READ mode will be selected in these cases with
// a warning indicating why it was chosen.
enum FileReadMode { READ, MAP };
string source; // Where you're reading from.
const FstHeader *header; // Pointer to FST header; if non-zero, use
// this info (don't read a stream header).
const SymbolTable *isymbols; // Pointer to input symbols; if non-zero, use
// this info (read and skip stream isymbols)
const SymbolTable *osymbols; // Pointer to output symbols; if non-zero, use
// this info (read and skip stream osymbols)
FileReadMode mode; // Read or map files (advisory, if possible)
bool read_isymbols; // Read isymbols, if any (default: true).
bool read_osymbols; // Read osymbols, if any (default: true).
explicit FstReadOptions(const string &source = "<unspecified>",
const FstHeader *header = nullptr,
const SymbolTable *isymbols = nullptr,
const SymbolTable *osymbols = nullptr);
explicit FstReadOptions(const string &source, const SymbolTable *isymbols,
const SymbolTable *osymbols = nullptr);
// Helper function to convert strings FileReadModes into their enum value.
static FileReadMode ReadMode(const string &mode);
// Outputs a debug string for the FstReadOptions object.
string DebugString() const;
};
struct FstWriteOptions {
string source; // Where you're writing to.
bool write_header; // Write the header?
bool write_isymbols; // Write input symbols?
bool write_osymbols; // Write output symbols?
bool align; // Write data aligned (may fail on pipes)?
bool stream_write; // Avoid seek operations in writing.
explicit FstWriteOptions(const string &source = "<unspecifed>",
bool write_header = true, bool write_isymbols = true,
bool write_osymbols = true,
bool align = FLAGS_fst_align,
bool stream_write = false)
: source(source),
write_header(write_header),
write_isymbols(write_isymbols),
write_osymbols(write_osymbols),
align(align),
stream_write(stream_write) {}
};
// Header class.
//
// This is the recommended file header representation.
class FstHeader {
public:
enum {
HAS_ISYMBOLS = 0x1, // Has input symbol table.
HAS_OSYMBOLS = 0x2, // Has output symbol table.
IS_ALIGNED = 0x4, // Memory-aligned (where appropriate).
} Flags;
FstHeader() : version_(0), flags_(0), properties_(0), start_(-1),
numstates_(0), numarcs_(0) {}
const string &FstType() const { return fsttype_; }
const string &ArcType() const { return arctype_; }
int32 Version() const { return version_; }
int32 GetFlags() const { return flags_; }
uint64 Properties() const { return properties_; }
int64 Start() const { return start_; }
int64 NumStates() const { return numstates_; }
int64 NumArcs() const { return numarcs_; }
void SetFstType(const string &type) { fsttype_ = type; }
void SetArcType(const string &type) { arctype_ = type; }
void SetVersion(int32 version) { version_ = version; }
void SetFlags(int32 flags) { flags_ = flags; }
void SetProperties(uint64 properties) { properties_ = properties; }
void SetStart(int64 start) { start_ = start; }
void SetNumStates(int64 numstates) { numstates_ = numstates; }
void SetNumArcs(int64 numarcs) { numarcs_ = numarcs; }
bool Read(std::istream &strm, const string &source,
bool rewind = false);
bool Write(std::ostream &strm, const string &source) const;
// Outputs a debug string for the FstHeader object.
string DebugString() const;
private:
string fsttype_; // E.g. "vector".
string arctype_; // E.g. "standard".
int32 version_; // Type version number.
int32 flags_; // File format bits.
uint64 properties_; // FST property bits.
int64 start_; // Start state.
int64 numstates_; // # of states.
int64 numarcs_; // # of arcs.
};
// Specifies matcher action.
enum MatchType {
MATCH_INPUT = 1, // Match input label.
MATCH_OUTPUT = 2, // Match output label.
MATCH_BOTH = 3, // Match input or output label.
MATCH_NONE = 4, // Match nothing.
MATCH_UNKNOWN = 5
}; // Otherwise, match type unknown.
constexpr int kNoLabel = -1; // Not a valid label.
constexpr int kNoStateId = -1; // Not a valid state ID.
// A generic FST, templated on the arc definition, with common-demoninator
// methods (use StateIterator and ArcIterator to iterate over its states and
// arcs).
template <class A>
class Fst {
public:
using Arc = A;
using StateId = typename Arc::StateId;
using Weight = typename Arc::Weight;
virtual ~Fst() {}
// Initial state.
virtual StateId Start() const = 0;
// State's final weight.
virtual Weight Final(StateId) const = 0;
// State's arc count.
virtual size_t NumArcs(StateId) const = 0;
// State's input epsilon count.
virtual size_t NumInputEpsilons(StateId) const = 0;
// State's output epsilon count.
virtual size_t NumOutputEpsilons(StateId) const = 0;
// Property bits. If test = false, return stored properties bits for mask
// (some possibly unknown); if test = true, return property bits for mask
// (computing o.w. unknown).
virtual uint64 Properties(uint64 mask, bool test) const = 0;
// FST type name.
virtual const string &Type() const = 0;
// Gets a copy of this Fst. The copying behaves as follows:
//
// (1) The copying is constant time if safe = false or if safe = true
// and is on an otherwise unaccessed FST.
//
// (2) If safe = true, the copy is thread-safe in that the original
// and copy can be safely accessed (but not necessarily mutated) by
// separate threads. For some FST types, 'Copy(true)' should only be
// called on an FST that has not otherwise been accessed. Behavior is
// otherwise undefined.
//
// (3) If a MutableFst is copied and then mutated, then the original is
// unmodified and vice versa (often by a copy-on-write on the initial
// mutation, which may not be constant time).
virtual Fst<Arc> *Copy(bool safe = false) const = 0;
// Reads an FST from an input stream; returns nullptr on error.
static Fst<Arc> *Read(std::istream &strm, const FstReadOptions &opts) {
FstReadOptions ropts(opts);
FstHeader hdr;
if (ropts.header) {
hdr = *opts.header;
} else {
if (!hdr.Read(strm, opts.source)) return nullptr;
ropts.header = &hdr;
}
const auto &fst_type = hdr.FstType();
const auto reader = FstRegister<Arc>::GetRegister()->GetReader(fst_type);
if (!reader) {
LOG(ERROR) << "Fst::Read: Unknown FST type " << fst_type
<< " (arc type = " << Arc::Type() << "): " << ropts.source;
return nullptr;
}
return reader(strm, ropts);
}
// Reads an FST from a file; returns nullptr on error. An empty filename
// results in reading from standard input.
static Fst<Arc> *Read(const string &filename) {
if (!filename.empty()) {
std::ifstream strm(filename,
std::ios_base::in | std::ios_base::binary);
if (!strm) {
LOG(ERROR) << "Fst::Read: Can't open file: " << filename;
return nullptr;
}
return Read(strm, FstReadOptions(filename));
} else {
return Read(std::cin, FstReadOptions("standard input"));
}
}
// Writes an FST to an output stream; returns false on error.
virtual bool Write(std::ostream &strm, const FstWriteOptions &opts) const {
LOG(ERROR) << "Fst::Write: No write stream method for " << Type()
<< " FST type";
return false;
}
// Writes an FST to a file; returns false on error; an empty filename
// results in writing to standard output.
virtual bool Write(const string &filename) const {
LOG(ERROR) << "Fst::Write: No write filename method for " << Type()
<< " FST type";
return false;
}
// Returns input label symbol table; return nullptr if not specified.
virtual const SymbolTable *InputSymbols() const = 0;
// Return output label symbol table; return nullptr if not specified.
virtual const SymbolTable *OutputSymbols() const = 0;
// For generic state iterator construction (not normally called directly by
// users). Does not copy the FST.
virtual void InitStateIterator(StateIteratorData<Arc> *data) const = 0;
// For generic arc iterator construction (not normally called directly by
// users). Does not copy the FST.
virtual void InitArcIterator(StateId s, ArcIteratorData<Arc> *data) const = 0;
// For generic matcher construction (not normally called directly by users).
// Does not copy the FST.
virtual MatcherBase<Arc> *InitMatcher(MatchType match_type) const;
protected:
bool WriteFile(const string &filename) const {
if (!filename.empty()) {
std::ofstream strm(filename,
std::ios_base::out | std::ios_base::binary);
if (!strm) {
LOG(ERROR) << "Fst::Write: Can't open file: " << filename;
return false;
}
bool val = Write(strm, FstWriteOptions(filename));
if (!val) LOG(ERROR) << "Fst::Write failed: " << filename;
return val;
} else {
return Write(std::cout, FstWriteOptions("standard output"));
}
}
};
// A useful alias when using StdArc.
using StdFst = Fst<StdArc>;
// State and arc iterator definitions.
//
// State iterator interface templated on the Arc definition; used for
// StateIterator specializations returned by the InitStateIterator FST method.
template <class Arc>
class StateIteratorBase {
public:
using StateId = typename Arc::StateId;
virtual ~StateIteratorBase() {}
// End of iterator?
virtual bool Done() const = 0;
// Returns current state (when !Done()).
virtual StateId Value() const = 0;
// Advances to next state (when !Done()).
virtual void Next() = 0;
// Resets to initial condition.
virtual void Reset() = 0;
};
// StateIterator initialization data.
template <class Arc>
struct StateIteratorData {
using StateId = typename Arc::StateId;
// Specialized iterator if non-zero.
StateIteratorBase<Arc> *base;
// Otherwise, the total number of states.
StateId nstates;
StateIteratorData() : base(nullptr), nstates(0) {}
StateIteratorData(const StateIteratorData &) = delete;
StateIteratorData &operator=(const StateIteratorData &) = delete;
};
// Generic state iterator, templated on the FST definition (a wrapper
// around a pointer to a specific one). Here is a typical use:
//
// for (StateIterator<StdFst> siter(fst);
// !siter.Done();
// siter.Next()) {
// StateId s = siter.Value();
// ...
// }
// There is no copying of the FST.
template <class FST>
class StateIterator {
public:
using Arc = typename FST::Arc;
using StateId = typename Arc::StateId;
explicit StateIterator(const FST &fst) : s_(0) {
fst.InitStateIterator(&data_);
}
~StateIterator() { delete data_.base; }
bool Done() const {
return data_.base ? data_.base->Done() : s_ >= data_.nstates;
}
StateId Value() const { return data_.base ? data_.base->Value() : s_; }
void Next() {
if (data_.base) {
data_.base->Next();
} else {
++s_;
}
}
void Reset() {
if (data_.base) {
data_.base->Reset();
} else {
s_ = 0;
}
}
private:
StateIteratorData<Arc> data_;
StateId s_;
};
// Flags to control the behavior on an arc iterator.
static constexpr uint32 kArcILabelValue =
0x0001; // Value() gives valid ilabel.
static constexpr uint32 kArcOLabelValue = 0x0002; // " " " olabel.
static constexpr uint32 kArcWeightValue = 0x0004; // " " " weight.
static constexpr uint32 kArcNextStateValue =
0x0008; // " " " nextstate.
static constexpr uint32 kArcNoCache = 0x0010; // No need to cache arcs.
static constexpr uint32 kArcValueFlags =
kArcILabelValue | kArcOLabelValue | kArcWeightValue | kArcNextStateValue;
static constexpr uint32 kArcFlags = kArcValueFlags | kArcNoCache;
// Arc iterator interface, templated on the arc definition; used for arc
// iterator specializations that are returned by the InitArcIterator FST method.
template <class Arc>
class ArcIteratorBase {
public:
using StateId = typename Arc::StateId;
virtual ~ArcIteratorBase() {}
// End of iterator?
virtual bool Done() const = 0;
// Returns current arc (when !Done()).
virtual const Arc &Value() const = 0;
// Advances to next arc (when !Done()).
virtual void Next() = 0;
// Returns current position.
virtual size_t Position() const = 0;
// Returns to initial condition.
virtual void Reset() = 0;
// Advances to arbitrary arc by position.
virtual void Seek(size_t) = 0;
// Returns current behavorial flags
virtual uint32 Flags() const = 0;
// Sets behavorial flags.
virtual void SetFlags(uint32, uint32) = 0;
};
// ArcIterator initialization data.
template <class Arc>
struct ArcIteratorData {
ArcIteratorData()
: base(nullptr), arcs(nullptr), narcs(0), ref_count(nullptr) {}
ArcIteratorData(const ArcIteratorData &) = delete;
ArcIteratorData &operator=(const ArcIteratorData &) = delete;
ArcIteratorBase<Arc> *base; // Specialized iterator if non-zero.
const Arc *arcs; // O.w. arcs pointer
size_t narcs; // ... and arc count.
int *ref_count; // ... and reference count if non-zero.
};
// Generic arc iterator, templated on the FST definition (a wrapper around a
// pointer to a specific one). Here is a typical use:
//
// for (ArcIterator<StdFst> aiter(fst, s);
// !aiter.Done();
// aiter.Next()) {
// StdArc &arc = aiter.Value();
// ...
// }
// There is no copying of the FST.
template <class FST>
class ArcIterator {
public:
using Arc = typename FST::Arc;
using StateId = typename Arc::StateId;
ArcIterator(const FST &fst, StateId s) : i_(0) {
fst.InitArcIterator(s, &data_);
}
explicit ArcIterator(const ArcIteratorData<Arc> &data) : data_(data), i_(0) {
if (data_.ref_count) ++(*data_.ref_count);
}
~ArcIterator() {
if (data_.base) {
delete data_.base;
} else if (data_.ref_count) {
--(*data_.ref_count);
}
}
bool Done() const {
return data_.base ? data_.base->Done() : i_ >= data_.narcs;
}
const Arc &Value() const {
return data_.base ? data_.base->Value() : data_.arcs[i_];
}
void Next() {
if (data_.base) {
data_.base->Next();
} else {
++i_;
}
}
void Reset() {
if (data_.base) {
data_.base->Reset();
} else {
i_ = 0;
}
}
void Seek(size_t a) {
if (data_.base) {
data_.base->Seek(a);
} else {
i_ = a;
}
}
size_t Position() const { return data_.base ? data_.base->Position() : i_; }
uint32 Flags() const {
if (data_.base) {
return data_.base->Flags();
} else {
return kArcValueFlags;
}
}
void SetFlags(uint32 flags, uint32 mask) {
if (data_.base) data_.base->SetFlags(flags, mask);
}
private:
ArcIteratorData<Arc> data_;
size_t i_;
};
} // namespace fst
// ArcIterator placement operator new and destroy function; new needs to be in
// the global namespace.
template <class FST>
void *operator new(size_t size,
fst::MemoryPool<fst::ArcIterator<FST>> *pool) {
return pool->Allocate();
}
namespace fst {
template <class FST>
void Destroy(ArcIterator<FST> *aiter, MemoryPool<ArcIterator<FST>> *pool) {
if (aiter) {
aiter->~ArcIterator<FST>();
pool->Free(aiter);
}
}
// Matcher definitions.
template <class Arc>
MatcherBase<Arc> *Fst<Arc>::InitMatcher(MatchType match_type) const {
return nullptr; // One should just use the default matcher.
}
// FST accessors, useful in high-performance applications.
namespace internal {
// General case, requires non-abstract, 'final' methods. Use for inlining.
template <class F>
inline typename F::Arc::Weight Final(const F &fst, typename F::Arc::StateId s) {
return fst.F::Final(s);
}
template <class F>
inline ssize_t NumArcs(const F &fst, typename F::Arc::StateId s) {
return fst.F::NumArcs(s);
}
template <class F>
inline ssize_t NumInputEpsilons(const F &fst, typename F::Arc::StateId s) {
return fst.F::NumInputEpsilons(s);
}
template <class F>
inline ssize_t NumOutputEpsilons(const F &fst, typename F::Arc::StateId s) {
return fst.F::NumOutputEpsilons(s);
}
// Fst<Arc> case, abstract methods.
template <class Arc>
inline typename Arc::Weight Final(const Fst<Arc> &fst,
typename Arc::StateId s) {
return fst.Final(s);
}
template <class Arc>
inline size_t NumArcs(const Fst<Arc> &fst, typename Arc::StateId s) {
return fst.NumArcs(s);
}
template <class Arc>
inline size_t NumInputEpsilons(const Fst<Arc> &fst, typename Arc::StateId s) {
return fst.NumInputEpsilons(s);
}
template <class Arc>
inline size_t NumOutputEpsilons(const Fst<Arc> &fst, typename Arc::StateId s) {
return fst.NumOutputEpsilons(s);
}
// FST implementation base.
//
// This is the recommended FST implementation base class. It will handle
// reference counts, property bits, type information and symbols.
//
// Users are discouraged, but not prohibited, from subclassing this outside the
// FST library.
template <class Arc>
class FstImpl {
public:
using StateId = typename Arc::StateId;
using Weight = typename Arc::Weight;
FstImpl() : properties_(0), type_("null") {}
FstImpl(const FstImpl<Arc> &impl)
: properties_(impl.properties_),
type_(impl.type_),
isymbols_(impl.isymbols_ ? impl.isymbols_->Copy() : nullptr),
osymbols_(impl.osymbols_ ? impl.osymbols_->Copy() : nullptr) {}
FstImpl(FstImpl<Arc> &&impl) noexcept;
virtual ~FstImpl() {}
FstImpl &operator=(const FstImpl<Arc> &impl) {
properties_ = impl.properties_;
type_ = impl.type_;
isymbols_ = impl.isymbols_ ? impl.isymbols_->Copy() : nullptr;
osymbols_ = impl.osymbols_ ? impl.osymbols_->Copy() : nullptr;
return *this;
}
FstImpl &operator=(FstImpl<Arc> &&impl) noexcept;
const string &Type() const { return type_; }
void SetType(const string &type) { type_ = type; }
virtual uint64 Properties() const { return properties_; }
virtual uint64 Properties(uint64 mask) const { return properties_ & mask; }
void SetProperties(uint64 props) {
properties_ &= kError; // kError can't be cleared.
properties_ |= props;
}
void SetProperties(uint64 props, uint64 mask) {
properties_ &= ~mask | kError; // kError can't be cleared.
properties_ |= props & mask;
}
// Allows (only) setting error bit on const FST implementations.
void SetProperties(uint64 props, uint64 mask) const {
if (mask != kError) {
FSTERROR() << "FstImpl::SetProperties() const: Can only set kError";
}
properties_ |= kError;
}
const SymbolTable *InputSymbols() const { return isymbols_.get(); }
const SymbolTable *OutputSymbols() const { return osymbols_.get(); }
SymbolTable *InputSymbols() { return isymbols_.get(); }
SymbolTable *OutputSymbols() { return osymbols_.get(); }
void SetInputSymbols(const SymbolTable *isyms) {
isymbols_.reset(isyms ? isyms->Copy() : nullptr);
}
void SetOutputSymbols(const SymbolTable *osyms) {
osymbols_.reset(osyms ? osyms->Copy() : nullptr);
}
// Reads header and symbols from input stream, initializes FST, and returns
// the header. If opts.header is non-null, skips reading and uses the option
// value instead. If opts.[io]symbols is non-null, reads in (if present), but
// uses the option value.
bool ReadHeader(std::istream &strm, const FstReadOptions &opts,
int min_version, FstHeader *hdr);
// Writes header and symbols to output stream. If opts.header is false, skips
// writing header. If opts.[io]symbols is false, skips writing those symbols.
// This method is needed for implementations that implement Write methods.
void WriteHeader(std::ostream &strm, const FstWriteOptions &opts,
int version, FstHeader *hdr) const {
if (opts.write_header) {
hdr->SetFstType(type_);
hdr->SetArcType(Arc::Type());
hdr->SetVersion(version);
hdr->SetProperties(properties_);
int32 file_flags = 0;
if (isymbols_ && opts.write_isymbols) {
file_flags |= FstHeader::HAS_ISYMBOLS;
}
if (osymbols_ && opts.write_osymbols) {
file_flags |= FstHeader::HAS_OSYMBOLS;
}
if (opts.align) file_flags |= FstHeader::IS_ALIGNED;
hdr->SetFlags(file_flags);
hdr->Write(strm, opts.source);
}
if (isymbols_ && opts.write_isymbols) isymbols_->Write(strm);
if (osymbols_ && opts.write_osymbols) osymbols_->Write(strm);
}
// Writes out header and symbols to output stream. If opts.header is false,
// skips writing header. If opts.[io]symbols is false, skips writing those
// symbols. `type` is the FST type being written. This method is used in the
// cross-type serialization methods Fst::WriteFst.
static void WriteFstHeader(const Fst<Arc> &fst, std::ostream &strm,
const FstWriteOptions &opts, int version,
const string &type, uint64 properties,
FstHeader *hdr) {
if (opts.write_header) {
hdr->SetFstType(type);
hdr->SetArcType(Arc::Type());
hdr->SetVersion(version);
hdr->SetProperties(properties);
int32 file_flags = 0;
if (fst.InputSymbols() && opts.write_isymbols) {
file_flags |= FstHeader::HAS_ISYMBOLS;
}
if (fst.OutputSymbols() && opts.write_osymbols) {
file_flags |= FstHeader::HAS_OSYMBOLS;
}
if (opts.align) file_flags |= FstHeader::IS_ALIGNED;
hdr->SetFlags(file_flags);
hdr->Write(strm, opts.source);
}
if (fst.InputSymbols() && opts.write_isymbols) {
fst.InputSymbols()->Write(strm);
}
if (fst.OutputSymbols() && opts.write_osymbols) {
fst.OutputSymbols()->Write(strm);
}
}
// In serialization routines where the header cannot be written until after
// the machine has been serialized, this routine can be called to seek to the
// beginning of the file an rewrite the header with updated fields. It
// repositions the file pointer back at the end of the file. Returns true on
// success, false on failure.
static bool UpdateFstHeader(const Fst<Arc> &fst, std::ostream &strm,
const FstWriteOptions &opts, int version,
const string &type, uint64 properties,
FstHeader *hdr, size_t header_offset) {
strm.seekp(header_offset);
if (!strm) {
LOG(ERROR) << "Fst::UpdateFstHeader: Write failed: " << opts.source;
return false;
}
WriteFstHeader(fst, strm, opts, version, type, properties, hdr);
if (!strm) {
LOG(ERROR) << "Fst::UpdateFstHeader: Write failed: " << opts.source;
return false;
}
strm.seekp(0, std::ios_base::end);
if (!strm) {
LOG(ERROR) << "Fst::UpdateFstHeader: Write failed: " << opts.source;
return false;
}
return true;
}
protected:
mutable uint64 properties_; // Property bits.
private:
string type_; // Unique name of FST class.
std::unique_ptr<SymbolTable> isymbols_;
std::unique_ptr<SymbolTable> osymbols_;
};
template <class Arc>
inline FstImpl<Arc>::FstImpl(FstImpl<Arc> &&) noexcept = default;
template <class Arc>
inline FstImpl<Arc> &FstImpl<Arc>::operator=(
FstImpl<Arc> &&) noexcept = default;
template <class Arc>
bool FstImpl<Arc>::ReadHeader(std::istream &strm, const FstReadOptions &opts,
int min_version, FstHeader *hdr) {
if (opts.header) {
*hdr = *opts.header;
} else if (!hdr->Read(strm, opts.source)) {
return false;
}
if (FLAGS_v >= 2) {
LOG(INFO) << "FstImpl::ReadHeader: source: " << opts.source
<< ", fst_type: " << hdr->FstType()
<< ", arc_type: " << Arc::Type()
<< ", version: " << hdr->Version()
<< ", flags: " << hdr->GetFlags();
}
if (hdr->FstType() != type_) {
LOG(ERROR) << "FstImpl::ReadHeader: FST not of type " << type_
<< ": " << opts.source;
return false;
}
if (hdr->ArcType() != Arc::Type()) {
LOG(ERROR) << "FstImpl::ReadHeader: Arc not of type " << Arc::Type()
<< ": " << opts.source;
return false;
}
if (hdr->Version() < min_version) {
LOG(ERROR) << "FstImpl::ReadHeader: Obsolete " << type_
<< " FST version: " << opts.source;
return false;
}
properties_ = hdr->Properties();
if (hdr->GetFlags() & FstHeader::HAS_ISYMBOLS) {
isymbols_.reset(SymbolTable::Read(strm, opts.source));
}
// Deletes input symbol table.
if (!opts.read_isymbols) SetInputSymbols(nullptr);
if (hdr->GetFlags() & FstHeader::HAS_OSYMBOLS) {
osymbols_.reset(SymbolTable::Read(strm, opts.source));
}
// Deletes output symbol table.
if (!opts.read_osymbols) SetOutputSymbols(nullptr);
if (opts.isymbols) {
isymbols_.reset(opts.isymbols->Copy());
}
if (opts.osymbols) {
osymbols_.reset(opts.osymbols->Copy());
}
return true;
}
} // namespace internal
template <class Arc>
uint64 TestProperties(const Fst<Arc> &fst, uint64 mask, uint64 *known);
// This is a helper class template useful for attaching an FST interface to
// its implementation, handling reference counting.
template <class Impl, class FST = Fst<typename Impl::Arc>>
class ImplToFst : public FST {
public:
using Arc = typename Impl::Arc;
using StateId = typename Arc::StateId;
using Weight = typename Arc::Weight;
StateId Start() const override { return impl_->Start(); }
Weight Final(StateId s) const override { return impl_->Final(s); }
size_t NumArcs(StateId s) const override { return impl_->NumArcs(s); }
size_t NumInputEpsilons(StateId s) const override {
return impl_->NumInputEpsilons(s);
}
size_t NumOutputEpsilons(StateId s) const override {
return impl_->NumOutputEpsilons(s);
}
uint64 Properties(uint64 mask, bool test) const override {
if (test) {
uint64 knownprops, testprops = TestProperties(*this, mask, &knownprops);
impl_->SetProperties(testprops, knownprops);
return testprops & mask;
} else {
return impl_->Properties(mask);
}
}
const string &Type() const override { return impl_->Type(); }
const SymbolTable *InputSymbols() const override {
return impl_->InputSymbols();
}
const SymbolTable *OutputSymbols() const override {
return impl_->OutputSymbols();
}
protected:
explicit ImplToFst(std::shared_ptr<Impl> impl) : impl_(std::move(impl)) {}
// This constructor presumes there is a copy constructor for the
// implementation.
ImplToFst(const ImplToFst<Impl, FST> &fst, bool safe) {
if (safe) {
impl_ = std::make_shared<Impl>(*(fst.impl_));
} else {
impl_ = fst.impl_;
}
}
ImplToFst() = delete;
ImplToFst(const ImplToFst<Impl, FST> &fst) : impl_(fst.impl_) {}
ImplToFst(ImplToFst<Impl, FST> &&fst) noexcept
: impl_(std::move(fst.impl_)) {
fst.impl_ = std::make_shared<Impl>();
}
ImplToFst<Impl, FST> &operator=(const ImplToFst<Impl, FST> &fst) {
impl_ = fst.impl_;
return *this;
}
ImplToFst<Impl, FST> &operator=(ImplToFst<Impl, FST> &&fst) noexcept {
if (this != &fst) {
impl_ = std::move(fst.impl_);
fst.impl_ = std::make_shared<Impl>();
}
return *this;
}
// Returns raw pointers to the shared object.
const Impl *GetImpl() const { return impl_.get(); }
Impl *GetMutableImpl() const { return impl_.get(); }
// Returns a ref-counted smart poiner to the implementation.
std::shared_ptr<Impl> GetSharedImpl() const { return impl_; }
bool Unique() const { return impl_.unique(); }
void SetImpl(std::shared_ptr<Impl> impl) { impl_ = std::move(impl); }
private:
template <class IFST, class OFST>
friend void Cast(const IFST &ifst, OFST *ofst);
std::shared_ptr<Impl> impl_;
};
// Converts FSTs by casting their implementations, where this makes sense
// (which excludes implementations with weight-dependent virtual methods).
// Must be a friend of the FST classes involved (currently the concrete FSTs:
// ConstFst, CompactFst, and VectorFst). This can only be safely used for arc
// types that have identical storage characteristics. As with an FST
// copy constructor and Copy() method, this is a constant time operation
// (but subject to copy-on-write if it is a MutableFst and modified).
template <class IFST, class OFST>
void Cast(const IFST &ifst, OFST *ofst) {
using OImpl = typename OFST::Impl;
ofst->impl_ = std::shared_ptr<OImpl>(ifst.impl_,
reinterpret_cast<OImpl *>(ifst.impl_.get()));
}
// FST serialization.
template <class Arc>
string FstToString(const Fst<Arc> &fst,
const FstWriteOptions &options =
FstWriteOptions("FstToString")) {
std::ostringstream ostrm;
fst.Write(ostrm, options);
return ostrm.str();
}
template <class Arc>
void FstToString(const Fst<Arc> &fst, string *result) {
*result = FstToString(fst);
}
template <class Arc>
void FstToString(const Fst<Arc> &fst, string *result,
const FstWriteOptions &options) {
*result = FstToString(fst, options);
}
template <class Arc>
Fst<Arc> *StringToFst(const string &s) {
std::istringstream istrm(s);
return Fst<Arc>::Read(istrm, FstReadOptions("StringToFst"));
}
} // namespace fst
#endif // FST_FST_H_