Skip to content

web search done right

mark the query parser in tantivy with #[cfg(test)] to ensure we don't accidentally use it instead of stracts

Mikkel Denker8af21448988b
diff --git a/crates/tantivy/examples/basic_search.rs b/crates/tantivy/examples/basic_search.rs
deleted file mode 100644
index 3c9c6ca8..00000000
--- a/crates/tantivy/examples/basic_search.rs
+++ /dev/null
@@ -1,239 +0,0 @@
-// # Basic Example
-//
-// This example covers the basic functionalities of
-// tantivy.
-//
-// We will :
-// - define our schema
-// - create an index in a directory
-// - index a few documents into our index
-// - search for the best document matching a basic query
-// - retrieve the best document's original content.
-
-// ---
-// Importing tantivy...
-use tantivy::collector::TopDocs;
-use tantivy::query::QueryParser;
-use tantivy::schema::*;
-use tantivy::{doc, Index, IndexWriter, ReloadPolicy};
-use tempfile::TempDir;
-
-fn main() -> tantivy::Result<()> {
- // Let's create a temporary directory for the
- // sake of this example
- let index_path = TempDir::new()?;
-
- // # Defining the schema
- //
- // The Tantivy index requires a very strict schema.
- // The schema declares which fields are in the index,
- // and for each field, its type and "the way it should
- // be indexed".
-
- // First we need to define a schema ...
- let mut schema_builder = Schema::builder();
-
- // Our first field is title.
- // We want full-text search for it, and we also want
- // to be able to retrieve the document after the search.
- //
- // `TEXT | STORED` is some syntactic sugar to describe
- // that.
- //
- // `TEXT` means the field should be tokenized and indexed,
- // along with its term frequency and term positions.
- //
- // `STORED` means that the field will also be saved
- // in a compressed, row-oriented key-value store.
- // This store is useful for reconstructing the
- // documents that were selected during the search phase.
- schema_builder.add_text_field("title", TEXT | STORED);
-
- // Our second field is body.
- // We want full-text search for it, but we do not
- // need to be able to be able to retrieve it
- // for our application.
- //
- // We can make our index lighter by omitting the `STORED` flag.
- schema_builder.add_text_field("body", TEXT);
-
- let schema = schema_builder.build();
-
- // # Indexing documents
- //
- // Let's create a brand new index.
- //
- // This will actually just save a meta.json
- // with our schema in the directory.
- let index = Index::create_in_dir(&index_path, schema.clone())?;
-
- // To insert a document we will need an index writer.
- // There must be only one writer at a time.
- // This single `IndexWriter` is already
- // multithreaded.
- //
- // Here we give tantivy a budget of `50MB`.
- // Using a bigger memory_arena for the indexer may increase
- // throughput, but 50 MB is already plenty.
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
-
- // Let's index our documents!
- // We first need a handle on the title and the body field.
-
- // ### Adding documents
- //
- // We can create a document manually, by setting the fields
- // one by one in a Document object.
- let title = schema.get_field("title").unwrap();
- let body = schema.get_field("body").unwrap();
-
- let mut old_man_doc = TantivyDocument::default();
- old_man_doc.add_text(title, "The Old Man and the Sea");
- old_man_doc.add_text(
- body,
- "He was an old man who fished alone in a skiff in the Gulf Stream and he had gone \
- eighty-four days now without taking a fish.",
- );
-
- // ... and add it to the `IndexWriter`.
- index_writer.add_document(old_man_doc)?;
-
- // For convenience, tantivy also comes with a macro to
- // reduce the boilerplate above.
- index_writer.add_document(doc!(
- title => "Of Mice and Men",
- body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
- bank and runs deep and green. The water is warm too, for it has slipped twinkling \
- over the yellow sands in the sunlight before reaching the narrow pool. On one \
- side of the river the golden foothill slopes curve up to the strong and rocky \
- Gabilan Mountains, but on the valley side the water is lined with trees—willows \
- fresh and green with every spring, carrying in their lower leaf junctures the \
- debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
- limbs and branches that arch over the pool"
- ))?;
-
- // Multivalued field just need to be repeated.
- index_writer.add_document(doc!(
- title => "Frankenstein",
- title => "The Modern Prometheus",
- body => "You will rejoice to hear that no disaster has accompanied the commencement of an \
- enterprise which you have regarded with such evil forebodings. I arrived here \
- yesterday, and my first task is to assure my dear sister of my welfare and \
- increasing confidence in the success of my undertaking."
- ))?;
-
- // This is an example, so we will only index 3 documents
- // here. You can check out tantivy's tutorial to index
- // the English wikipedia. Tantivy's indexing is rather fast.
- // Indexing 5 million articles of the English wikipedia takes
- // around 3 minutes on my computer!
-
- // ### Committing
- //
- // At this point our documents are not searchable.
- //
- //
- // We need to call `.commit()` explicitly to force the
- // `index_writer` to finish processing the documents in the queue,
- // flush the current index to the disk, and advertise
- // the existence of new documents.
- //
- // This call is blocking.
- index_writer.commit()?;
-
- // If `.commit()` returns correctly, then all of the
- // documents that have been added are guaranteed to be
- // persistently indexed.
- //
- // In the scenario of a crash or a power failure,
- // tantivy behaves as if it has rolled back to its last
- // commit.
-
- // # Searching
- //
- // ### Searcher
- //
- // A reader is required first in order to search an index.
- // It acts as a `Searcher` pool that reloads itself,
- // depending on a `ReloadPolicy`.
- //
- // For a search server you will typically create one reader for the entire lifetime of your
- // program, and acquire a new searcher for every single request.
- //
- // In the code below, we rely on the 'ON_COMMIT' policy: the reader
- // will reload the index automatically after each commit.
- let reader = index
- .reader_builder()
- .reload_policy(ReloadPolicy::OnCommitWithDelay)
- .try_into()?;
-
- // We now need to acquire a searcher.
- //
- // A searcher points to a snapshotted, immutable version of the index.
- //
- // Some search experience might require more than
- // one query. Using the same searcher ensures that all of these queries will run on the
- // same version of the index.
- //
- // Acquiring a `searcher` is very cheap.
- //
- // You should acquire a searcher every time you start processing a request and
- // and release it right after your query is finished.
- let searcher = reader.searcher();
-
- // ### Query
-
- // The query parser can interpret human queries.
- // Here, if the user does not specify which
- // field they want to search, tantivy will search
- // in both title and body.
- let query_parser = QueryParser::for_index(&index, vec![title, body]);
-
- // `QueryParser` may fail if the query is not in the right
- // format. For user facing applications, this can be a problem.
- // A ticket has been opened regarding this problem.
- let query = query_parser.parse_query("sea whale")?;
-
- // A query defines a set of documents, as
- // well as the way they should be scored.
- //
- // A query created by the query parser is scored according
- // to a metric called Tf-Idf, and will consider
- // any document matching at least one of our terms.
-
- // ### Collectors
- //
- // We are not interested in all of the documents but
- // only in the top 10. Keeping track of our top 10 best documents
- // is the role of the `TopDocs` collector.
-
- // We can now perform our query.
- let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
-
- // The actual documents still need to be
- // retrieved from Tantivy's store.
- //
- // Since the body field was not configured as stored,
- // the document returned will only contain
- // a title.
- for (_score, doc_address) in top_docs {
- let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
- println!("{}", retrieved_doc.to_json(&schema));
- }
-
- // We can also get an explanation to understand
- // how a found document got its score.
- let query = query_parser.parse_query("title:sea^20 body:whale^70")?;
-
- let (_score, doc_address) = searcher
- .search(&query, &TopDocs::with_limit(1))?
- .into_iter()
- .next()
- .unwrap();
-
- let explanation = query.explain(&searcher, doc_address)?;
-
- println!("{}", explanation.to_pretty_json());
-
- Ok(())
-}
diff --git a/crates/tantivy/examples/custom_collector.rs b/crates/tantivy/examples/custom_collector.rs
deleted file mode 100644
index 6c6e3475..00000000
--- a/crates/tantivy/examples/custom_collector.rs
+++ /dev/null
@@ -1,186 +0,0 @@
-// # Custom collector example
-//
-// This example shows how you can implement your own
-// collector. As an example, we will compute a collector
-// that computes the standard deviation of a given fast field.
-//
-// Of course, you can have a look at the tantivy's built-in collectors
-// such as the `CountCollector` for more examples.
-
-use tantivy::columnar::Column;
-// ---
-// Importing tantivy...
-use tantivy::collector::{Collector, SegmentCollector};
-use tantivy::index::SegmentReader;
-use tantivy::query::QueryParser;
-use tantivy::schema::{Schema, FAST, INDEXED, TEXT};
-use tantivy::{doc, Index, IndexWriter, Score};
-
-#[derive(Default)]
-struct Stats {
- count: usize,
- sum: f64,
- squared_sum: f64,
-}
-
-impl Stats {
- pub fn count(&self) -> usize {
- self.count
- }
-
- pub fn mean(&self) -> f64 {
- self.sum / (self.count as f64)
- }
-
- fn square_mean(&self) -> f64 {
- self.squared_sum / (self.count as f64)
- }
-
- pub fn standard_deviation(&self) -> f64 {
- let mean = self.mean();
- (self.square_mean() - mean * mean).sqrt()
- }
-
- fn non_zero_count(self) -> Option<Stats> {
- if self.count == 0 {
- None
- } else {
- Some(self)
- }
- }
-}
-
-struct StatsCollector {
- field: String,
-}
-
-impl StatsCollector {
- fn with_field(field: String) -> StatsCollector {
- StatsCollector { field }
- }
-}
-
-impl Collector for StatsCollector {
- // That's the type of our result.
- // Our standard deviation will be a float.
- type Fruit = Option<Stats>;
-
- type Child = StatsSegmentCollector;
-
- fn for_segment(
- &self,
- _segment_local_id: u32,
- segment_reader: &SegmentReader,
- ) -> tantivy::Result<StatsSegmentCollector> {
- let fast_field_reader = segment_reader.fast_fields().u64(&self.field)?;
- Ok(StatsSegmentCollector {
- fast_field_reader,
- stats: Stats::default(),
- })
- }
-
- fn requires_scoring(&self) -> bool {
- // this collector does not care about score.
- false
- }
-
- fn merge_fruits(&self, segment_stats: Vec<Option<Stats>>) -> tantivy::Result<Option<Stats>> {
- let mut stats = Stats::default();
- for segment_stats in segment_stats.into_iter().flatten() {
- stats.count += segment_stats.count;
- stats.sum += segment_stats.sum;
- stats.squared_sum += segment_stats.squared_sum;
- }
- Ok(stats.non_zero_count())
- }
-}
-
-struct StatsSegmentCollector {
- fast_field_reader: Column,
- stats: Stats,
-}
-
-impl SegmentCollector for StatsSegmentCollector {
- type Fruit = Option<Stats>;
-
- fn collect(&mut self, doc: u32, _score: Score) {
- // Since we know the values are single value, we could call `first_or_default_col` on the
- // column and fetch single values.
- for value in self.fast_field_reader.values_for_doc(doc) {
- let value = value as f64;
- self.stats.count += 1;
- self.stats.sum += value;
- self.stats.squared_sum += value * value;
- }
- }
-
- fn harvest(self) -> <Self as SegmentCollector>::Fruit {
- self.stats.non_zero_count()
- }
-}
-
-fn main() -> tantivy::Result<()> {
- // # Defining the schema
- //
- // The Tantivy index requires a very strict schema.
- // The schema declares which fields are in the index,
- // and for each field, its type and "the way it should
- // be indexed".
-
- // first we need to define a schema ...
- let mut schema_builder = Schema::builder();
-
- // We'll assume a fictional index containing
- // products, and with a name, a description, and a price.
- let product_name = schema_builder.add_text_field("name", TEXT);
- let product_description = schema_builder.add_text_field("description", TEXT);
- let price = schema_builder.add_u64_field("price", INDEXED | FAST);
- let schema = schema_builder.build();
-
- // # Indexing documents
- //
- // Lets index a bunch of fake documents for the sake of
- // this example.
- let index = Index::create_in_ram(schema);
-
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
- index_writer.add_document(doc!(
- product_name => "Super Broom 2000",
- product_description => "While it is ok for short distance travel, this broom \
- was designed quiditch. It will up your game.",
- price => 30_200u64
- ))?;
- index_writer.add_document(doc!(
- product_name => "Turbulobroom",
- product_description => "You might have heard of this broom before : it is the sponsor of the Wales team.\
- You'll enjoy its sharp turns, and rapid acceleration",
- price => 29_240u64
- ))?;
- index_writer.add_document(doc!(
- product_name => "Broomio",
- product_description => "Great value for the price. This broom is a market favorite",
- price => 21_240u64
- ))?;
- index_writer.add_document(doc!(
- product_name => "Whack a Mole",
- product_description => "Prime quality bat.",
- price => 5_200u64
- ))?;
- index_writer.commit()?;
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let query_parser = QueryParser::for_index(&index, vec![product_name, product_description]);
-
- // here we want to search for `broom` and use `StatsCollector` on the hits.
- let query = query_parser.parse_query("broom")?;
- if let Some(stats) =
- searcher.search(&query, &StatsCollector::with_field("price".to_string()))?
- {
- println!("count: {}", stats.count());
- println!("mean: {}", stats.mean());
- println!("standard deviation: {}", stats.standard_deviation());
- }
-
- Ok(())
-}
diff --git a/crates/tantivy/examples/custom_tokenizer.rs b/crates/tantivy/examples/custom_tokenizer.rs
deleted file mode 100644
index 6ec6047c..00000000
--- a/crates/tantivy/examples/custom_tokenizer.rs
+++ /dev/null
@@ -1,111 +0,0 @@
-// # Defining a tokenizer pipeline
-//
-// In this example, we'll see how to define a tokenizer
-// by creating a custom `NgramTokenizer`.
-use tantivy::collector::TopDocs;
-use tantivy::query::QueryParser;
-use tantivy::schema::*;
-use tantivy::tokenizer::NgramTokenizer;
-use tantivy::{doc, Index, IndexWriter};
-
-fn main() -> tantivy::Result<()> {
- // # Defining the schema
- //
- // The Tantivy index requires a very strict schema.
- // The schema declares which fields are in the index,
- // and for each field, its type and "the way it should
- // be indexed".
-
- // first we need to define a schema ...
- let mut schema_builder = Schema::builder();
-
- // Our first field is title.
- // In this example we want to use NGram searching
- // we will set that to 3 characters, so any three
- // char in the title should be findable.
- let text_field_indexing = TextFieldIndexing::default()
- .set_tokenizer("ngram3")
- .set_index_option(IndexRecordOption::WithFreqsAndPositions);
- let text_options = TextOptions::default()
- .set_indexing_options(text_field_indexing)
- .set_stored();
- let title = schema_builder.add_text_field("title", text_options);
-
- // Our second field is body.
- // We want full-text search for it, but we do not
- // need to be able to be able to retrieve it
- // for our application.
- //
- // We can make our index lighter by omitting the `STORED` flag.
- let body = schema_builder.add_text_field("body", TEXT);
-
- let schema = schema_builder.build();
-
- // # Indexing documents
- //
- // Let's create a brand new index.
- // To simplify we will work entirely in RAM.
- // This is not what you want in reality, but it is very useful
- // for your unit tests... Or this example.
- let index = Index::create_in_ram(schema.clone());
-
- // here we are registering our custom tokenizer
- // this will store tokens of 3 characters each
- index
- .tokenizers()
- .register("ngram3", NgramTokenizer::new(3, 3, false).unwrap());
-
- // To insert document we need an index writer.
- // There must be only one writer at a time.
- // This single `IndexWriter` is already
- // multithreaded.
- //
- // Here we use a buffer of 50MB per thread. Using a bigger
- // memory arena for the indexer can increase its throughput.
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
- index_writer.add_document(doc!(
- title => "The Old Man and the Sea",
- body => "He was an old man who fished alone in a skiff in the Gulf Stream and \
- he had gone eighty-four days now without taking a fish."
- ))?;
- index_writer.add_document(doc!(
- title => "Of Mice and Men",
- body => r#"A few miles south of Soledad, the Salinas River drops in close to the hillside
- bank and runs deep and green. The water is warm too, for it has slipped twinkling
- over the yellow sands in the sunlight before reaching the narrow pool. On one
- side of the river the golden foothill slopes curve up to the strong and rocky
- Gabilan Mountains, but on the valley side the water is lined with trees—willows
- fresh and green with every spring, carrying in their lower leaf junctures the
- debris of the winter’s flooding; and sycamores with mottled, white, recumbent
- limbs and branches that arch over the pool"#
- ))?;
- index_writer.add_document(doc!(
- title => "Frankenstein",
- body => r#"You will rejoice to hear that no disaster has accompanied the commencement of an
- enterprise which you have regarded with such evil forebodings. I arrived here
- yesterday, and my first task is to assure my dear sister of my welfare and
- increasing confidence in the success of my undertaking."#
- ))?;
- index_writer.commit()?;
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
-
- // The query parser can interpret human queries.
- // Here, if the user does not specify which
- // field they want to search, tantivy will search
- // in both title and body.
- let query_parser = QueryParser::for_index(&index, vec![title, body]);
-
- // here we want to get a hit on the 'ken' in Frankenstein
- let query = query_parser.parse_query("ken")?;
-
- let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
-
- for (_, doc_address) in top_docs {
- let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
- println!("{}", retrieved_doc.to_json(&schema));
- }
-
- Ok(())
-}
diff --git a/crates/tantivy/examples/date_time_field.rs b/crates/tantivy/examples/date_time_field.rs
deleted file mode 100644
index c00a22f0..00000000
--- a/crates/tantivy/examples/date_time_field.rs
+++ /dev/null
@@ -1,77 +0,0 @@
-// # DateTime field example
-//
-// This example shows how the DateTime field can be used
-
-use tantivy::collector::TopDocs;
-use tantivy::query::QueryParser;
-use tantivy::schema::{DateOptions, Document, Schema, Value, INDEXED, STORED, STRING};
-use tantivy::{Index, IndexWriter, TantivyDocument};
-
-fn main() -> tantivy::Result<()> {
- // # Defining the schema
- let mut schema_builder = Schema::builder();
- let opts = DateOptions::from(INDEXED)
- .set_stored()
- .set_fast()
- .set_precision(tantivy::schema::DateTimePrecision::Seconds);
- // Add `occurred_at` date field type
- let occurred_at = schema_builder.add_date_field("occurred_at", opts);
- let event_type = schema_builder.add_text_field("event", STRING | STORED);
- let schema = schema_builder.build();
-
- // # Indexing documents
- let index = Index::create_in_ram(schema.clone());
-
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
- // The dates are passed as string in the RFC3339 format
- let doc = TantivyDocument::parse_json(
- &schema,
- r#"{
- "occurred_at": "2022-06-22T12:53:50.53Z",
- "event": "pull-request"
- }"#,
- )?;
- index_writer.add_document(doc)?;
- let doc = TantivyDocument::parse_json(
- &schema,
- r#"{
- "occurred_at": "2022-06-22T13:00:00.22Z",
- "event": "comment"
- }"#,
- )?;
- index_writer.add_document(doc)?;
- index_writer.commit()?;
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
-
- // # Search
- let query_parser = QueryParser::for_index(&index, vec![event_type]);
- {
- // Simple exact search on the date
- let query = query_parser.parse_query("occurred_at:\"2022-06-22T12:53:50.53Z\"")?;
- let count_docs = searcher.search(&*query, &TopDocs::with_limit(5))?;
- assert_eq!(count_docs.len(), 1);
- }
- {
- // Range query on the date field
- let query = query_parser
- .parse_query(r#"occurred_at:[2022-06-22T12:58:00Z TO 2022-06-23T00:00:00Z}"#)?;
- let count_docs = searcher.search(&*query, &TopDocs::with_limit(4))?;
- assert_eq!(count_docs.len(), 1);
- for (_score, doc_address) in count_docs {
- let retrieved_doc = searcher.doc::<TantivyDocument>(doc_address)?;
- assert!(retrieved_doc
- .get_first(occurred_at)
- .unwrap()
- .as_value()
- .as_datetime()
- .is_some(),);
- assert_eq!(
- retrieved_doc.to_json(&schema),
- r#"{"event":["comment"],"occurred_at":["2022-06-22T13:00:00.22Z"]}"#
- );
- }
- }
- Ok(())
-}
diff --git a/crates/tantivy/examples/deleting_updating_documents.rs b/crates/tantivy/examples/deleting_updating_documents.rs
deleted file mode 100644
index ac9859a4..00000000
--- a/crates/tantivy/examples/deleting_updating_documents.rs
+++ /dev/null
@@ -1,144 +0,0 @@
-// # Deleting and Updating (?) documents
-//
-// This example explains how to delete and update documents.
-// In fact there is actually no such thing as an update in tantivy.
-//
-// To update a document, you need to delete a document and then reinsert
-// its new version.
-//
-// ---
-// Importing tantivy...
-use tantivy::collector::TopDocs;
-use tantivy::query::TermQuery;
-use tantivy::schema::*;
-use tantivy::{doc, Index, IndexReader, IndexWriter};
-
-// A simple helper function to fetch a single document
-// given its id from our index.
-// It will be helpful to check our work.
-fn extract_doc_given_isbn(
- reader: &IndexReader,
- isbn_term: &Term,
-) -> tantivy::Result<Option<TantivyDocument>> {
- let searcher = reader.searcher();
-
- // This is the simplest query you can think of.
- // It matches all of the documents containing a specific term.
- //
- // The second argument is here to tell we don't care about decoding positions,
- // or term frequencies.
- let term_query = TermQuery::new(isbn_term.clone(), IndexRecordOption::Basic);
- let top_docs = searcher.search(&term_query, &TopDocs::with_limit(1))?;
-
- if let Some((_score, doc_address)) = top_docs.first() {
- let doc = searcher.doc(*doc_address)?;
- Ok(Some(doc))
- } else {
- // no doc matching this ID.
- Ok(None)
- }
-}
-
-fn main() -> tantivy::Result<()> {
- // # Defining the schema
- //
- // Check out the *basic_search* example if this makes
- // small sense to you.
- let mut schema_builder = Schema::builder();
-
- // Tantivy does not really have a notion of primary id.
- // This may change in the future.
- //
- // Still, we can create a `isbn` field and use it as an id. This
- // field can be `u64` or a `text`, depending on your use case.
- // It just needs to be indexed.
- //
- // If it is `text`, let's make sure to keep it `raw` and let's avoid
- // running any text processing on it.
- // This is done by associating this field to the tokenizer named `raw`.
- // Rather than building our
- // [`TextOptions`](//docs.rs/tantivy/~0/tantivy/schema/struct.TextOptions.html) manually, We
- // use the `STRING` shortcut. `STRING` stands for indexed (without term frequency or positions)
- // and untokenized.
- //
- // Because we also want to be able to see this `id` in our returned documents,
- // we also mark the field as stored.
- let isbn = schema_builder.add_text_field("isbn", STRING | STORED);
- let title = schema_builder.add_text_field("title", TEXT | STORED);
- let schema = schema_builder.build();
-
- let index = Index::create_in_ram(schema.clone());
-
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
-
- // Let's add a couple of documents, for the sake of the example.
- let mut old_man_doc = TantivyDocument::default();
- old_man_doc.add_text(title, "The Old Man and the Sea");
- index_writer.add_document(doc!(
- isbn => "978-0099908401",
- title => "The old Man and the see"
- ))?;
- index_writer.add_document(doc!(
- isbn => "978-0140177398",
- title => "Of Mice and Men",
- ))?;
- index_writer.add_document(doc!(
- title => "Frankentein", //< Oops there is a typo here.
- isbn => "978-9176370711",
- ))?;
- index_writer.commit()?;
- let reader = index.reader()?;
-
- let frankenstein_isbn = Term::from_field_text(isbn, "978-9176370711");
-
- // Oops our frankenstein doc seems misspelled
- let frankenstein_doc_misspelled = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
- assert_eq!(
- frankenstein_doc_misspelled.to_json(&schema),
- r#"{"isbn":["978-9176370711"],"title":["Frankentein"]}"#,
- );
-
- // # Update = Delete + Insert
- //
- // Here we will want to update the typo in the `Frankenstein` book.
- //
- // Tantivy does not handle updates directly, we need to delete
- // and reinsert the document.
- //
- // This can be complicated as it means you need to have access
- // to the entire document. It is good practise to integrate tantivy
- // with a key value store for this reason.
- //
- // To remove one of the document, we just call `delete_term`
- // on its id.
- //
- // Note that `tantivy` does nothing to enforce the idea that
- // there is only one document associated with this id.
- //
- // Also you might have noticed that we apply the delete before
- // having committed. This does not matter really...
- index_writer.delete_term(frankenstein_isbn.clone());
-
- // We now need to reinsert our document without the typo.
- index_writer.add_document(doc!(
- title => "Frankenstein",
- isbn => "978-9176370711",
- ))?;
-
- // You are guaranteed that your clients will only observe your index in
- // the state it was in after a commit.
- // In this example, your search engine will at no point be missing the *Frankenstein* document.
- // Everything happened as if the document was updated.
- index_writer.commit()?;
- // We reload our searcher to make our change available to clients.
- reader.reload()?;
-
- // No more typo!
- let frankenstein_new_doc = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
- assert_eq!(
- frankenstein_new_doc.to_json(&schema),
- r#"{"isbn":["978-9176370711"],"title":["Frankenstein"]}"#,
- );
-
- Ok(())
-}
diff --git a/crates/tantivy/examples/faceted_search.rs b/crates/tantivy/examples/faceted_search.rs
deleted file mode 100644
index 216f55f7..00000000
--- a/crates/tantivy/examples/faceted_search.rs
+++ /dev/null
@@ -1,114 +0,0 @@
-// # Faceted Search
-//
-// This example covers the faceted search functionalities of
-// tantivy.
-//
-// We will :
-// - define a text field "name" in our schema
-// - define a facet field "classification" in our schema
-// - create an index in memory
-// - index few documents with respective facets in our index
-// - search and count the number of documents that the classifications start the facet "/Felidae"
-// - Search the facet "/Felidae/Pantherinae" and count the number of documents that the
-// classifications include the facet.
-//
-// ---
-// Importing tantivy...
-use tantivy::collector::FacetCollector;
-use tantivy::query::{AllQuery, TermQuery};
-use tantivy::schema::*;
-use tantivy::{doc, Index, IndexWriter};
-
-fn main() -> tantivy::Result<()> {
- // Let's create a temporary directory for the sake of this example
- let mut schema_builder = Schema::builder();
-
- let name = schema_builder.add_text_field("name", TEXT | STORED);
- // this is our faceted field: its scientific classification
- let classification = schema_builder.add_facet_field("classification", FacetOptions::default());
-
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
-
- let mut index_writer: IndexWriter = index.writer(30_000_000)?;
-
- // For convenience, tantivy also comes with a macro to
- // reduce the boilerplate above.
- index_writer.add_document(doc!(
- name => "Cat",
- classification => Facet::from("/Felidae/Felinae/Felis")
- ))?;
- index_writer.add_document(doc!(
- name => "Canada lynx",
- classification => Facet::from("/Felidae/Felinae/Lynx")
- ))?;
- index_writer.add_document(doc!(
- name => "Cheetah",
- classification => Facet::from("/Felidae/Felinae/Acinonyx")
- ))?;
- index_writer.add_document(doc!(
- name => "Tiger",
- classification => Facet::from("/Felidae/Pantherinae/Panthera")
- ))?;
- index_writer.add_document(doc!(
- name => "Lion",
- classification => Facet::from("/Felidae/Pantherinae/Panthera")
- ))?;
- index_writer.add_document(doc!(
- name => "Jaguar",
- classification => Facet::from("/Felidae/Pantherinae/Panthera")
- ))?;
- index_writer.add_document(doc!(
- name => "Sunda clouded leopard",
- classification => Facet::from("/Felidae/Pantherinae/Neofelis")
- ))?;
- index_writer.add_document(doc!(
- name => "Fossa",
- classification => Facet::from("/Eupleridae/Cryptoprocta")
- ))?;
- index_writer.commit()?;
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- {
- let mut facet_collector = FacetCollector::for_field("classification");
- facet_collector.add_facet("/Felidae");
- let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
- // This lists all of the facet counts, right below "/Felidae".
- let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae").collect();
- assert_eq!(
- facets,
- vec![
- (&Facet::from("/Felidae/Felinae"), 3),
- (&Facet::from("/Felidae/Pantherinae"), 4),
- ]
- );
- }
-
- // Facets are also searchable.
- //
- // For instance a common UI pattern is to allow the user someone to click on a facet link
- // (e.g: `Pantherinae`) to drill down and filter the current result set with this subfacet.
- //
- // The search would then look as follows.
-
- // Check the reference doc for different ways to create a `Facet` object.
- {
- let facet = Facet::from("/Felidae/Pantherinae");
- let facet_term = Term::from_facet(classification, &facet);
- let facet_term_query = TermQuery::new(facet_term, IndexRecordOption::Basic);
- let mut facet_collector = FacetCollector::for_field("classification");
- facet_collector.add_facet("/Felidae/Pantherinae");
- let facet_counts = searcher.search(&facet_term_query, &facet_collector)?;
- let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae/Pantherinae").collect();
- assert_eq!(
- facets,
- vec![
- (&Facet::from("/Felidae/Pantherinae/Neofelis"), 1),
- (&Facet::from("/Felidae/Pantherinae/Panthera"), 3),
- ]
- );
- }
-
- Ok(())
-}
diff --git a/crates/tantivy/examples/faceted_search_with_tweaked_score.rs b/crates/tantivy/examples/faceted_search_with_tweaked_score.rs
deleted file mode 100644
index d21a1c3d..00000000
--- a/crates/tantivy/examples/faceted_search_with_tweaked_score.rs
+++ /dev/null
@@ -1,104 +0,0 @@
-// # Faceted Search With Tweak Score
-//
-// This example covers the faceted search functionalities of
-// tantivy.
-//
-// We will :
-// - define a text field "name" in our schema
-// - define a facet field "classification" in our schema
-
-use std::collections::HashSet;
-
-use tantivy::collector::TopDocs;
-use tantivy::query::BooleanQuery;
-use tantivy::schema::*;
-use tantivy::{doc, DocId, Index, IndexWriter, Score, SegmentReader};
-
-fn main() -> tantivy::Result<()> {
- let mut schema_builder = Schema::builder();
-
- let title = schema_builder.add_text_field("title", STORED);
- let ingredient = schema_builder.add_facet_field("ingredient", FacetOptions::default());
-
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
-
- let mut index_writer: IndexWriter = index.writer(30_000_000)?;
-
- index_writer.add_document(doc!(
- title => "Fried egg",
- ingredient => Facet::from("/ingredient/egg"),
- ingredient => Facet::from("/ingredient/oil"),
- ))?;
- index_writer.add_document(doc!(
- title => "Scrambled egg",
- ingredient => Facet::from("/ingredient/egg"),
- ingredient => Facet::from("/ingredient/butter"),
- ingredient => Facet::from("/ingredient/milk"),
- ingredient => Facet::from("/ingredient/salt"),
- ))?;
- index_writer.add_document(doc!(
- title => "Egg rolls",
- ingredient => Facet::from("/ingredient/egg"),
- ingredient => Facet::from("/ingredient/garlic"),
- ingredient => Facet::from("/ingredient/salt"),
- ingredient => Facet::from("/ingredient/oil"),
- ingredient => Facet::from("/ingredient/tortilla-wrap"),
- ingredient => Facet::from("/ingredient/mushroom"),
- ))?;
- index_writer.commit()?;
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- {
- let facets = [
- Facet::from("/ingredient/egg"),
- Facet::from("/ingredient/oil"),
- Facet::from("/ingredient/garlic"),
- Facet::from("/ingredient/mushroom"),
- ];
- let query = BooleanQuery::new_multiterms_query(
- facets
- .iter()
- .map(|key| Term::from_facet(ingredient, key))
- .collect(),
- );
- let top_docs_by_custom_score =
- // Call TopDocs with a custom tweak score
- TopDocs::with_limit(2).tweak_score(move |segment_reader: &SegmentReader| {
- let ingredient_reader = segment_reader.facet_reader("ingredient").unwrap();
- let facet_dict = ingredient_reader.facet_dict();
-
- let query_ords: HashSet<u64> = facets
- .iter()
- .filter_map(|key| facet_dict.term_ord(key.encoded_str()).unwrap())
- .collect();
-
- move |doc: DocId, original_score: Score| {
- // Update the original score with a tweaked score
- let missing_ingredients = ingredient_reader
- .facet_ords(doc)
- .filter(|ord| !query_ords.contains(ord))
- .count();
- let tweak = 1.0 / 4_f32.powi(missing_ingredients as i32);
-
- original_score * tweak
- }
- });
- let top_docs = searcher.search(&query, &top_docs_by_custom_score)?;
-
- let titles: Vec<String> = top_docs
- .iter()
- .map(|(_, doc_id)| {
- searcher
- .doc::<TantivyDocument>(*doc_id)
- .unwrap()
- .get_first(title)
- .and_then(|v| v.as_str().map(|el| el.to_string()))
- .unwrap()
- })
- .collect();
- assert_eq!(titles, vec!["Fried egg", "Egg rolls"]);
- }
- Ok(())
-}
diff --git a/crates/tantivy/examples/fuzzy_search.rs b/crates/tantivy/examples/fuzzy_search.rs
deleted file mode 100644
index ab681a7a..00000000
--- a/crates/tantivy/examples/fuzzy_search.rs
+++ /dev/null
@@ -1,167 +0,0 @@
-// # Basic Example
-//
-// This example covers the basic functionalities of
-// tantivy.
-//
-// We will :
-// - define our schema
-// - create an index in a directory
-// - index a few documents into our index
-// - search for the best document matching a basic query
-// - retrieve the best document's original content.
-// ---
-// Importing tantivy...
-use tantivy::collector::{Count, TopDocs};
-use tantivy::query::FuzzyTermQuery;
-use tantivy::schema::*;
-use tantivy::{doc, Index, IndexWriter, ReloadPolicy};
-use tempfile::TempDir;
-
-fn main() -> tantivy::Result<()> {
- // Let's create a temporary directory for the
- // sake of this example
- let index_path = TempDir::new()?;
-
- // # Defining the schema
- //
- // The Tantivy index requires a very strict schema.
- // The schema declares which fields are in the index,
- // and for each field, its type and "the way it should
- // be indexed".
-
- // First we need to define a schema ...
- let mut schema_builder = Schema::builder();
-
- // Our first field is title.
- // We want full-text search for it, and we also want
- // to be able to retrieve the document after the search.
- //
- // `TEXT | STORED` is some syntactic sugar to describe
- // that.
- //
- // `TEXT` means the field should be tokenized and indexed,
- // along with its term frequency and term positions.
- //
- // `STORED` means that the field will also be saved
- // in a compressed, row-oriented key-value store.
- // This store is useful for reconstructing the
- // documents that were selected during the search phase.
- let title = schema_builder.add_text_field("title", TEXT | STORED);
-
- let schema = schema_builder.build();
-
- // # Indexing documents
- //
- // Let's create a brand new index.
- //
- // This will actually just save a meta.json
- // with our schema in the directory.
- let index = Index::create_in_dir(&index_path, schema.clone())?;
-
- // To insert a document we will need an index writer.
- // There must be only one writer at a time.
- // This single `IndexWriter` is already
- // multithreaded.
- //
- // Here we give tantivy a budget of `50MB`.
- // Using a bigger memory_arena for the indexer may increase
- // throughput, but 50 MB is already plenty.
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
-
- // Let's index our documents!
- // We first need a handle on the title and the body field.
-
- // ### Adding documents
- //
- index_writer.add_document(doc!(
- title => "The Name of the Wind",
- ))?;
- index_writer.add_document(doc!(
- title => "The Diary of Muadib",
- ))?;
- index_writer.add_document(doc!(
- title => "A Dairy Cow",
- ))?;
- index_writer.add_document(doc!(
- title => "The Diary of a Young Girl",
- ))?;
- index_writer.commit()?;
-
- // ### Committing
- //
- // At this point our documents are not searchable.
- //
- //
- // We need to call `.commit()` explicitly to force the
- // `index_writer` to finish processing the documents in the queue,
- // flush the current index to the disk, and advertise
- // the existence of new documents.
- //
- // This call is blocking.
- index_writer.commit()?;
-
- // If `.commit()` returns correctly, then all of the
- // documents that have been added are guaranteed to be
- // persistently indexed.
- //
- // In the scenario of a crash or a power failure,
- // tantivy behaves as if it has rolled back to its last
- // commit.
-
- // # Searching
- //
- // ### Searcher
- //
- // A reader is required first in order to search an index.
- // It acts as a `Searcher` pool that reloads itself,
- // depending on a `ReloadPolicy`.
- //
- // For a search server you will typically create one reader for the entire lifetime of your
- // program, and acquire a new searcher for every single request.
- //
- // In the code below, we rely on the 'ON_COMMIT' policy: the reader
- // will reload the index automatically after each commit.
- let reader = index
- .reader_builder()
- .reload_policy(ReloadPolicy::OnCommitWithDelay)
- .try_into()?;
-
- // We now need to acquire a searcher.
- //
- // A searcher points to a snapshotted, immutable version of the index.
- //
- // Some search experience might require more than
- // one query. Using the same searcher ensures that all of these queries will run on the
- // same version of the index.
- //
- // Acquiring a `searcher` is very cheap.
- //
- // You should acquire a searcher every time you start processing a request and
- // and release it right after your query is finished.
- let searcher = reader.searcher();
-
- // ### FuzzyTermQuery
- {
- let term = Term::from_field_text(title, "Diary");
- let query = FuzzyTermQuery::new(term, 2, true);
-
- let (top_docs, count) = searcher
- .search(&query, &(TopDocs::with_limit(5), Count))
- .unwrap();
- assert_eq!(count, 3);
- assert_eq!(top_docs.len(), 3);
- for (score, doc_address) in top_docs {
- // Note that the score is not lower for the fuzzy hit.
- // There's an issue open for that: https://github.com/quickwit-oss/tantivy/issues/563
- let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
- println!("score {score:?} doc {}", retrieved_doc.to_json(&schema));
- // score 1.0 doc {"title":["The Diary of Muadib"]}
- //
- // score 1.0 doc {"title":["The Diary of a Young Girl"]}
- //
- // score 1.0 doc {"title":["A Dairy Cow"]}
- }
- }
-
- Ok(())
-}
diff --git a/crates/tantivy/examples/index_from_multiple_threads.rs b/crates/tantivy/examples/index_from_multiple_threads.rs
deleted file mode 100644
index 288d306d..00000000
--- a/crates/tantivy/examples/index_from_multiple_threads.rs
+++ /dev/null
@@ -1,104 +0,0 @@
-// # Indexing from different threads.
-//
-// It is fairly common to have to index from different threads.
-// Tantivy forbids to create more than one `IndexWriter` at a time.
-//
-// This `IndexWriter` itself has its own multithreaded layer, so managing your own
-// indexing threads will not help. However, it can still be useful for some applications.
-//
-// For instance, if preparing documents to send to tantivy before indexing is the bottleneck of
-// your application, it is reasonable to have multiple threads.
-//
-// Another very common reason to want to index from multiple threads, is implementing a webserver
-// with CRUD capabilities. The server framework will most likely handle request from
-// different threads.
-//
-// The recommended way to address both of these use case is to wrap your `IndexWriter` into a
-// `Arc<RwLock<IndexWriter>>`.
-//
-// While this is counterintuitive, adding and deleting documents do not require mutability
-// over the `IndexWriter`, so several threads will be able to do this operation concurrently.
-//
-// The example below does not represent an actual real-life use case (who would spawn thread to
-// index a single document?), but aims at demonstrating the mechanism that makes indexing
-// from several threads possible.
-
-// ---
-// Importing tantivy...
-use std::sync::{Arc, RwLock};
-use std::thread;
-use std::time::Duration;
-
-use tantivy::schema::{Schema, STORED, TEXT};
-use tantivy::{doc, Index, IndexWriter, Opstamp, TantivyError};
-
-fn main() -> tantivy::Result<()> {
- // # Defining the schema
- let mut schema_builder = Schema::builder();
- let title = schema_builder.add_text_field("title", TEXT | STORED);
- let body = schema_builder.add_text_field("body", TEXT);
- let schema = schema_builder.build();
-
- let index = Index::create_in_ram(schema);
- let index_writer: Arc<RwLock<IndexWriter>> = Arc::new(RwLock::new(index.writer(50_000_000)?));
-
- // # First indexing thread.
- let index_writer_clone_1 = index_writer.clone();
- thread::spawn(move || {
- // we index 100 times the document... for the sake of the example.
- for i in 0..100 {
- let opstamp = index_writer_clone_1
- .read().unwrap() //< A read lock is sufficient here.
- .add_document(
- doc!(
- title => "Of Mice and Men",
- body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
- bank and runs deep and green. The water is warm too, for it has slipped twinkling \
- over the yellow sands in the sunlight before reaching the narrow pool. On one \
- side of the river the golden foothill slopes curve up to the strong and rocky \
- Gabilan Mountains, but on the valley side the water is lined with trees—willows \
- fresh and green with every spring, carrying in their lower leaf junctures the \
- debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
- limbs and branches that arch over the pool"
- ))?;
- println!("add doc {i} from thread 1 - opstamp {opstamp}");
- thread::sleep(Duration::from_millis(20));
- }
- Result::<(), TantivyError>::Ok(())
- });
-
- // # Second indexing thread.
- let index_writer_clone_2 = index_writer.clone();
- // For convenience, tantivy also comes with a macro to
- // reduce the boilerplate above.
- thread::spawn(move || {
- // we index 100 times the document... for the sake of the example.
- for i in 0..100 {
- // A read lock is sufficient here.
- let opstamp = {
- let index_writer_rlock = index_writer_clone_2.read().unwrap();
- index_writer_rlock.add_document(doc!(
- title => "Manufacturing consent",
- body => "Some great book description..."
- ))?
- };
- println!("add doc {i} from thread 2 - opstamp {opstamp}");
- thread::sleep(Duration::from_millis(10));
- }
- Result::<(), TantivyError>::Ok(())
- });
-
- // # In the main thread, we commit 10 times, once every 500ms.
- for _ in 0..10 {
- let opstamp: Opstamp = {
- // Committing or rollbacking on the other hand requires write lock. This will block
- // other threads.
- let mut index_writer_wlock = index_writer.write().unwrap();
- index_writer_wlock.commit()?
- };
- println!("committed with opstamp {opstamp}");
- thread::sleep(Duration::from_millis(500));
- }
-
- Ok(())
-}
diff --git a/crates/tantivy/examples/index_with_json.rs b/crates/tantivy/examples/index_with_json.rs
deleted file mode 100644
index 5869234f..00000000
--- a/crates/tantivy/examples/index_with_json.rs
+++ /dev/null
@@ -1,40 +0,0 @@
-use tantivy::schema::*;
-
-// # Document from json
-//
-// For convenience, `Document` can be parsed directly from json.
-fn main() -> tantivy::Result<()> {
- // Let's first define a schema and an index.
- // Check out the basic example if this is confusing to you.
- //
- // first we need to define a schema ...
- let mut schema_builder = Schema::builder();
- schema_builder.add_text_field("title", TEXT | STORED);
- schema_builder.add_text_field("body", TEXT);
- schema_builder.add_u64_field("year", INDEXED);
- let schema = schema_builder.build();
-
- // Let's assume we have a json-serialized document.
- let mice_and_men_doc_json = r#"{
- "title": "Of Mice and Men",
- "year": 1937
- }"#;
-
- // We can parse our document
- let _mice_and_men_doc = TantivyDocument::parse_json(&schema, mice_and_men_doc_json)?;
-
- // Multi-valued field are allowed, they are
- // expressed in JSON by an array.
- // The following document has two titles.
- let frankenstein_json = r#"{
- "title": ["Frankenstein", "The Modern Prometheus"],
- "year": 1818
- }"#;
- let _frankenstein_doc = TantivyDocument::parse_json(&schema, frankenstein_json)?;
-
- // Note that the schema is saved in your index directory.
- //
- // As a result, Indexes are aware of their schema, and you can use this feature
- // just by opening an existing `Index`, and calling `index.schema()..parse_document(json)`.
- Ok(())
-}
diff --git a/crates/tantivy/examples/integer_range_search.rs b/crates/tantivy/examples/integer_range_search.rs
deleted file mode 100644
index 10f2015c..00000000
--- a/crates/tantivy/examples/integer_range_search.rs
+++ /dev/null
@@ -1,35 +0,0 @@
-// # Searching a range on an indexed int field.
-//
-// Below is an example of creating an indexed integer field in your schema
-// You can use RangeQuery to get a Count of all occurrences in a given range.
-use tantivy::collector::Count;
-use tantivy::query::RangeQuery;
-use tantivy::schema::{Schema, INDEXED};
-use tantivy::{doc, Index, IndexWriter, Result};
-
-fn main() -> Result<()> {
- // For the sake of simplicity, this schema will only have 1 field
- let mut schema_builder = Schema::builder();
-
- // `INDEXED` is a short-hand to indicate that our field should be "searchable".
- let year_field = schema_builder.add_u64_field("year", INDEXED);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let reader = index.reader()?;
- {
- let mut index_writer: IndexWriter = index.writer_with_num_threads(1, 6_000_000)?;
- for year in 1950u64..2019u64 {
- index_writer.add_document(doc!(year_field => year))?;
- }
- index_writer.commit()?;
- // The index will be a range of years
- }
- reader.reload()?;
- let searcher = reader.searcher();
- // The end is excluded i.e. here we are searching up to 1969
- let docs_in_the_sixties = RangeQuery::new_u64("year".to_string(), 1960..1970);
- // Uses a Count collector to sum the total number of docs in the range
- let num_60s_books = searcher.search(&docs_in_the_sixties, &Count)?;
- assert_eq!(num_60s_books, 10);
- Ok(())
-}
diff --git a/crates/tantivy/examples/ip_field.rs b/crates/tantivy/examples/ip_field.rs
deleted file mode 100644
index 17caf55e..00000000
--- a/crates/tantivy/examples/ip_field.rs
+++ /dev/null
@@ -1,110 +0,0 @@
-// # IP Address example
-//
-// This example shows how the ip field can be used
-// with IpV6 and IpV4.
-
-use tantivy::collector::{Count, TopDocs};
-use tantivy::query::QueryParser;
-use tantivy::schema::{Schema, FAST, INDEXED, STORED, STRING};
-use tantivy::{Index, IndexWriter, TantivyDocument};
-
-fn main() -> tantivy::Result<()> {
- // # Defining the schema
- // We set the IP field as `INDEXED`, so it can be searched
- // `FAST` will create a fast field. The fast field will be used to execute search queries.
- // `FAST` is not a requirement for range queries, it can also be executed on the inverted index
- // which is created by `INDEXED`.
- let mut schema_builder = Schema::builder();
- let event_type = schema_builder.add_text_field("event_type", STRING | STORED);
- let ip = schema_builder.add_ip_addr_field("ip", STORED | INDEXED | FAST);
- let schema = schema_builder.build();
-
- // # Indexing documents
- let index = Index::create_in_ram(schema.clone());
-
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
-
- // ### IPv4
- // Adding documents that contain an IPv4 address. Notice that the IP addresses are passed as
- // `String`. Since the field is of type ip, we parse the IP address from the string and store it
- // internally as IPv6.
- let doc = TantivyDocument::parse_json(
- &schema,
- r#"{
- "ip": "192.168.0.33",
- "event_type": "login"
- }"#,
- )?;
- index_writer.add_document(doc)?;
- let doc = TantivyDocument::parse_json(
- &schema,
- r#"{
- "ip": "192.168.0.80",
- "event_type": "checkout"
- }"#,
- )?;
- index_writer.add_document(doc)?;
- // ### IPv6
- // Adding a document that contains an IPv6 address.
- let doc = TantivyDocument::parse_json(
- &schema,
- r#"{
- "ip": "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
- "event_type": "checkout"
- }"#,
- )?;
-
- index_writer.add_document(doc)?;
- // Commit will create a segment containing our documents.
- index_writer.commit()?;
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
-
- // # Search
- // Range queries on IPv4. Since we created a fast field, the fast field will be used to execute
- // the search.
- // ### Range Queries
- let query_parser = QueryParser::for_index(&index, vec![event_type, ip]);
- {
- // Inclusive range queries
- let query = query_parser.parse_query("ip:[192.168.0.80 TO 192.168.0.100]")?;
- let count_docs = searcher.search(&*query, &TopDocs::with_limit(5))?;
- assert_eq!(count_docs.len(), 1);
- }
- {
- // Exclusive range queries
- let query = query_parser.parse_query("ip:{192.168.0.80 TO 192.168.1.100]")?;
- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2))?;
- assert_eq!(count_docs.len(), 0);
- }
- {
- // Find docs with IP addresses smaller equal 192.168.1.100
- let query = query_parser.parse_query("ip:[* TO 192.168.1.100]")?;
- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2))?;
- assert_eq!(count_docs.len(), 2);
- }
- {
- // Find docs with IP addresses smaller than 192.168.1.100
- let query = query_parser.parse_query("ip:[* TO 192.168.1.100}")?;
- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2))?;
- assert_eq!(count_docs.len(), 2);
- }
-
- // ### Exact Queries
- // Exact search on IPv4.
- {
- let query = query_parser.parse_query("ip:192.168.0.80")?;
- let count_docs = searcher.search(&*query, &Count)?;
- assert_eq!(count_docs, 1);
- }
- // Exact search on IPv6.
- // IpV6 addresses need to be quoted because they contain `:`
- {
- let query = query_parser.parse_query("ip:\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\"")?;
- let count_docs = searcher.search(&*query, &Count)?;
- assert_eq!(count_docs, 1);
- }
-
- Ok(())
-}
diff --git a/crates/tantivy/examples/iterating_docs_and_positions.rs b/crates/tantivy/examples/iterating_docs_and_positions.rs
deleted file mode 100644
index b5827661..00000000
--- a/crates/tantivy/examples/iterating_docs_and_positions.rs
+++ /dev/null
@@ -1,136 +0,0 @@
-// # Iterating docs and positions.
-//
-// At its core of tantivy, relies on a data structure
-// called an inverted index.
-//
-// This example shows how to manually iterate through
-// the list of documents containing a term, getting
-// its term frequency, and accessing its positions.
-
-use tantivy::postings::Postings;
-// ---
-// Importing tantivy...
-use tantivy::schema::*;
-use tantivy::{doc, DocSet, Index, IndexWriter, TERMINATED};
-
-fn main() -> tantivy::Result<()> {
- // We first create a schema for the sake of the
- // example. Check the `basic_search` example for more information.
- let mut schema_builder = Schema::builder();
-
- // For this example, we need to make sure to index positions for our title
- // field. `TEXT` precisely does this.
- let title = schema_builder.add_text_field("title", TEXT | STORED);
- let schema = schema_builder.build();
-
- let index = Index::create_in_ram(schema);
-
- let mut index_writer: IndexWriter = index.writer_with_num_threads(1, 50_000_000)?;
- index_writer.add_document(doc!(title => "The Old Man and the Sea"))?;
- index_writer.add_document(doc!(title => "Of Mice and Men"))?;
- index_writer.add_document(doc!(title => "The modern Promotheus"))?;
- index_writer.commit()?;
-
- let reader = index.reader()?;
-
- let searcher = reader.searcher();
-
- // A tantivy index is actually a collection of segments.
- // Similarly, a searcher just wraps a list `segment_reader`.
- //
- // (Because we indexed a very small number of documents over one thread
- // there is actually only one segment here, but let's iterate through the list
- // anyway)
- for segment_reader in searcher.segment_readers() {
- // A segment contains different data structure.
- // Inverted index stands for the combination of
- // - the term dictionary
- // - the inverted lists associated with each terms and their positions
- let inverted_index = segment_reader.inverted_index(title)?;
-
- // A `Term` is a text token associated with a field.
- // Let's go through all docs containing the term `title:the` and access their position
- let term_the = Term::from_field_text(title, "the");
-
- // This segment posting object is like a cursor over the documents matching the term.
- // The `IndexRecordOption` arguments tells tantivy we will be interested in both term
- // frequencies and positions.
- //
- // If you don't need all this information, you may get better performance by decompressing
- // less information.
- if let Some(mut segment_postings) =
- inverted_index.read_postings(&term_the, IndexRecordOption::WithFreqsAndPositions)?
- {
- // this buffer will be used to request for positions
- let mut positions: Vec<u32> = Vec::with_capacity(100);
- let mut doc_id = segment_postings.doc();
- while doc_id != TERMINATED {
- // This MAY contains deleted documents as well.
- if segment_reader.is_deleted(doc_id) {
- doc_id = segment_postings.advance();
- continue;
- }
-
- // the number of time the term appears in the document.
- let term_freq: u32 = segment_postings.term_freq();
- // accessing positions is slightly expensive and lazy, do not request
- // for them if you don't need them for some documents.
- segment_postings.positions(&mut positions);
-
- // By definition we should have `term_freq` positions.
- assert_eq!(positions.len(), term_freq as usize);
-
- // This prints:
- // ```
- // Doc 0: TermFreq 2: [0, 4]
- // Doc 2: TermFreq 1: [0]
- // ```
- println!("Doc {doc_id}: TermFreq {term_freq}: {positions:?}");
- doc_id = segment_postings.advance();
- }
- }
- }
-
- // A `Term` is a text token associated with a field.
- // Let's go through all docs containing the term `title:the` and access their position
- let term_the = Term::from_field_text(title, "the");
-
- // Some other powerful operations (especially `.skip_to`) may be useful to consume these
- // posting lists rapidly.
- // You can check for them in the [`DocSet`](https://docs.rs/tantivy/~0/tantivy/trait.DocSet.html) trait
- // and the [`Postings`](https://docs.rs/tantivy/~0/tantivy/trait.Postings.html) trait
-
- // Also, for some VERY specific high performance use case like an OLAP analysis of logs,
- // you can get better performance by accessing directly the blocks of doc ids.
- for segment_reader in searcher.segment_readers() {
- // A segment contains different data structure.
- // Inverted index stands for the combination of
- // - the term dictionary
- // - the inverted lists associated with each terms and their positions
- let inverted_index = segment_reader.inverted_index(title)?;
-
- // This segment posting object is like a cursor over the documents matching the term.
- // The `IndexRecordOption` arguments tells tantivy we will be interested in both term
- // frequencies and positions.
- //
- // If you don't need all this information, you may get better performance by decompressing
- // less information.
- if let Some(mut block_segment_postings) =
- inverted_index.read_block_postings(&term_the, IndexRecordOption::Basic)?
- {
- loop {
- let docs = block_segment_postings.docs();
- if docs.is_empty() {
- break;
- }
- // Once again these docs MAY contains deleted documents as well.
- let docs = block_segment_postings.docs();
- // Prints `Docs [0, 2].`
- println!("Docs {docs:?}");
- block_segment_postings.advance();
- }
- }
- }
-
- Ok(())
-}
diff --git a/crates/tantivy/examples/json_field.rs b/crates/tantivy/examples/json_field.rs
deleted file mode 100644
index 334ea10e..00000000
--- a/crates/tantivy/examples/json_field.rs
+++ /dev/null
@@ -1,107 +0,0 @@
-// # Json field example
-//
-// This example shows how the json field can be used
-// to make tantivy partially schemaless by setting it as
-// default query parser field.
-
-use tantivy::collector::{Count, TopDocs};
-use tantivy::query::QueryParser;
-use tantivy::schema::{Schema, FAST, STORED, STRING, TEXT};
-use tantivy::{Index, IndexWriter, TantivyDocument};
-
-fn main() -> tantivy::Result<()> {
- // # Defining the schema
- let mut schema_builder = Schema::builder();
- schema_builder.add_date_field("timestamp", FAST | STORED);
- let event_type = schema_builder.add_text_field("event_type", STRING | STORED);
- let attributes = schema_builder.add_json_field("attributes", STORED | TEXT);
- let schema = schema_builder.build();
-
- // # Indexing documents
- let index = Index::create_in_ram(schema.clone());
-
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
- let doc = TantivyDocument::parse_json(
- &schema,
- r#"{
- "timestamp": "2022-02-22T23:20:50.53Z",
- "event_type": "click",
- "attributes": {
- "target": "submit-button",
- "cart": {"product_id": 103},
- "description": "the best vacuum cleaner ever"
- }
- }"#,
- )?;
- index_writer.add_document(doc)?;
- let doc = TantivyDocument::parse_json(
- &schema,
- r#"{
- "timestamp": "2022-02-22T23:20:51.53Z",
- "event_type": "click",
- "attributes": {
- "target": "submit-button",
- "cart": {"product_id": 133},
- "description": "das keyboard",
- "event_type": "holiday-sale"
- }
- }"#,
- )?;
- index_writer.add_document(doc)?;
- index_writer.commit()?;
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
-
- // # Default fields: event_type and attributes
- // By setting attributes as a default field it allows omitting attributes itself, e.g. "target",
- // instead of "attributes.target"
- let query_parser = QueryParser::for_index(&index, vec![event_type, attributes]);
- {
- let query = query_parser.parse_query("target:submit-button")?;
- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2))?;
- assert_eq!(count_docs.len(), 2);
- }
- {
- let query = query_parser.parse_query("target:submit")?;
- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2))?;
- assert_eq!(count_docs.len(), 2);
- }
- {
- let query = query_parser.parse_query("cart.product_id:103")?;
- let count_docs = searcher.search(&*query, &Count)?;
- assert_eq!(count_docs, 1);
- }
- {
- let query = query_parser.parse_query("click AND cart.product_id:133")?;
- let hits = searcher.search(&*query, &TopDocs::with_limit(2))?;
- assert_eq!(hits.len(), 1);
- }
- {
- // The sub-fields in the json field marked as default field still need to be explicitly
- // addressed
- let query = query_parser.parse_query("click AND 133")?;
- let hits = searcher.search(&*query, &TopDocs::with_limit(2))?;
- assert_eq!(hits.len(), 0);
- }
- {
- // Default json fields are ignored if they collide with the schema
- let query = query_parser.parse_query("event_type:holiday-sale")?;
- let hits = searcher.search(&*query, &TopDocs::with_limit(2))?;
- assert_eq!(hits.len(), 0);
- }
- // # Query via full attribute path
- {
- // This only searches in our schema's `event_type` field
- let query = query_parser.parse_query("event_type:click")?;
- let hits = searcher.search(&*query, &TopDocs::with_limit(2))?;
- assert_eq!(hits.len(), 2);
- }
- {
- // Default json fields can still be accessed by full path
- let query = query_parser.parse_query("attributes.event_type:holiday-sale")?;
- let hits = searcher.search(&*query, &TopDocs::with_limit(2))?;
- assert_eq!(hits.len(), 1);
- }
- Ok(())
-}
diff --git a/crates/tantivy/examples/phrase_prefix_search.rs b/crates/tantivy/examples/phrase_prefix_search.rs
deleted file mode 100644
index 24239dc7..00000000
--- a/crates/tantivy/examples/phrase_prefix_search.rs
+++ /dev/null
@@ -1,83 +0,0 @@
-use tantivy::collector::TopDocs;
-use tantivy::query::QueryParser;
-use tantivy::schema::*;
-use tantivy::{doc, Index, IndexWriter, ReloadPolicy, Result};
-use tempfile::TempDir;
-
-fn main() -> Result<()> {
- let index_path = TempDir::new()?;
-
- let mut schema_builder = Schema::builder();
- schema_builder.add_text_field("title", TEXT | STORED);
- schema_builder.add_text_field("body", TEXT);
- let schema = schema_builder.build();
-
- let title = schema.get_field("title").unwrap();
- let body = schema.get_field("body").unwrap();
-
- let index = Index::create_in_dir(&index_path, schema)?;
-
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
-
- index_writer.add_document(doc!(
- title => "The Old Man and the Sea",
- body => "He was an old man who fished alone in a skiff in the Gulf Stream and he had gone \
- eighty-four days now without taking a fish.",
- ))?;
-
- index_writer.add_document(doc!(
- title => "Of Mice and Men",
- body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
- bank and runs deep and green. The water is warm too, for it has slipped twinkling \
- over the yellow sands in the sunlight before reaching the narrow pool. On one \
- side of the river the golden foothill slopes curve up to the strong and rocky \
- Gabilan Mountains, but on the valley side the water is lined with trees—willows \
- fresh and green with every spring, carrying in their lower leaf junctures the \
- debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
- limbs and branches that arch over the pool"
- ))?;
-
- // Multivalued field just need to be repeated.
- index_writer.add_document(doc!(
- title => "Frankenstein",
- title => "The Modern Prometheus",
- body => "You will rejoice to hear that no disaster has accompanied the commencement of an \
- enterprise which you have regarded with such evil forebodings. I arrived here \
- yesterday, and my first task is to assure my dear sister of my welfare and \
- increasing confidence in the success of my undertaking."
- ))?;
-
- index_writer.commit()?;
-
- let reader = index
- .reader_builder()
- .reload_policy(ReloadPolicy::OnCommitWithDelay)
- .try_into()?;
-
- let searcher = reader.searcher();
-
- let query_parser = QueryParser::for_index(&index, vec![title, body]);
- // This will match documents containing the phrase "in the"
- // followed by some word starting with "su",
- // i.e. it will match "in the sunlight" and "in the success",
- // but not "in the Gulf Stream".
- let query = query_parser.parse_query("\"in the su\"*")?;
-
- let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
- let mut titles = top_docs
- .into_iter()
- .map(|(_score, doc_address)| {
- let doc = searcher.doc::<TantivyDocument>(doc_address)?;
- let title = doc
- .get_first(title)
- .and_then(|v| v.as_str())
- .unwrap()
- .to_owned();
- Ok(title)
- })
- .collect::<Result<Vec<_>>>()?;
- titles.sort_unstable();
- assert_eq!(titles, ["Frankenstein", "Of Mice and Men"]);
-
- Ok(())
-}
diff --git a/crates/tantivy/examples/pre_tokenized_text.rs b/crates/tantivy/examples/pre_tokenized_text.rs
deleted file mode 100644
index 4cd4e930..00000000
--- a/crates/tantivy/examples/pre_tokenized_text.rs
+++ /dev/null
@@ -1,136 +0,0 @@
-// # Pre-tokenized text example
-//
-// This example shows how to use pre-tokenized text. Sometimes you might
-// want to index and search through text which is already split into
-// tokens by some external tool.
-//
-// In this example we will:
-// - use tantivy tokenizer to create tokens and load them directly into tantivy,
-// - import tokenized text straight from json,
-// - perform a search on documents with pre-tokenized text
-
-use tantivy::collector::{Count, TopDocs};
-use tantivy::query::TermQuery;
-use tantivy::schema::*;
-use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, TokenStream, Tokenizer};
-use tantivy::{doc, Index, IndexWriter, ReloadPolicy};
-use tempfile::TempDir;
-
-fn pre_tokenize_text(text: &str) -> Vec<Token> {
- let mut tokenizer = SimpleTokenizer::default();
- let mut token_stream = tokenizer.token_stream(text);
- let mut tokens = vec![];
- while token_stream.advance() {
- tokens.push(token_stream.token().clone());
- }
- tokens
-}
-
-fn main() -> tantivy::Result<()> {
- let index_path = TempDir::new()?;
-
- let mut schema_builder = Schema::builder();
-
- schema_builder.add_text_field("title", TEXT | STORED);
- schema_builder.add_text_field("body", TEXT);
-
- let schema = schema_builder.build();
-
- let index = Index::create_in_dir(&index_path, schema.clone())?;
-
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
-
- // We can create a document manually, by setting the fields
- // one by one in a Document object.
- let title = schema.get_field("title").unwrap();
- let body = schema.get_field("body").unwrap();
-
- let title_text = "The Old Man and the Sea";
- let body_text = "He was an old man who fished alone in a skiff in the Gulf Stream";
-
- // Content of our first document
- // We create `PreTokenizedString` which contains original text and vector of tokens
- let title_tok = PreTokenizedString {
- text: String::from(title_text),
- tokens: pre_tokenize_text(title_text),
- };
-
- println!(
- "Original text: \"{}\" and tokens: {:?}",
- title_tok.text, title_tok.tokens
- );
-
- let body_tok = PreTokenizedString {
- text: String::from(body_text),
- tokens: pre_tokenize_text(body_text),
- };
-
- // Now lets create a document and add our `PreTokenizedString`
- let old_man_doc = doc!(title => title_tok, body => body_tok);
-
- // ... now let's just add it to the IndexWriter
- index_writer.add_document(old_man_doc)?;
-
- // Pretokenized text can also be fed as JSON
- let short_man_json = r#"{
- "title":[{
- "text":"The Old Man",
- "tokens":[
- {"offset_from":0,"offset_to":3,"position":0,"text":"The","position_length":1},
- {"offset_from":4,"offset_to":7,"position":1,"text":"Old","position_length":1},
- {"offset_from":8,"offset_to":11,"position":2,"text":"Man","position_length":1}
- ]
- }]
- }"#;
-
- let short_man_doc = TantivyDocument::parse_json(&schema, short_man_json)?;
-
- index_writer.add_document(short_man_doc)?;
-
- // Let's commit changes
- index_writer.commit()?;
-
- // ... and now is the time to query our index
-
- let reader = index
- .reader_builder()
- .reload_policy(ReloadPolicy::OnCommitWithDelay)
- .try_into()?;
-
- let searcher = reader.searcher();
-
- // We want to get documents with token "Man", we will use TermQuery to do it
- // Using PreTokenizedString means the tokens are stored as is avoiding stemming
- // and lowercasing, which preserves full words in their original form
- let query = TermQuery::new(
- Term::from_field_text(title, "Man"),
- IndexRecordOption::Basic,
- );
-
- let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count))?;
-
- assert_eq!(count, 2);
-
- // Now let's print out the results.
- // Note that the tokens are not stored along with the original text
- // in the document store
- for (_score, doc_address) in top_docs {
- let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
- println!("{}", retrieved_doc.to_json(&schema));
- }
-
- // In contrary to the previous query, when we search for the "man" term we
- // should get no results, as it's not one of the indexed tokens. SimpleTokenizer
- // only splits text on whitespace / punctuation.
-
- let query = TermQuery::new(
- Term::from_field_text(title, "man"),
- IndexRecordOption::Basic,
- );
-
- let (_top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count))?;
-
- assert_eq!(count, 0);
-
- Ok(())
-}
diff --git a/crates/tantivy/examples/snippet.rs b/crates/tantivy/examples/snippet.rs
deleted file mode 100644
index 31bd2c16..00000000
--- a/crates/tantivy/examples/snippet.rs
+++ /dev/null
@@ -1,83 +0,0 @@
-// # Snippet example
-//
-// This example shows how to return a representative snippet of
-// your hit result.
-// Snippet are an extracted of a target document, and returned in HTML format.
-// The keyword searched by the user are highlighted with a `<b>` tag.
-
-// ---
-// Importing tantivy...
-use tantivy::collector::TopDocs;
-use tantivy::query::QueryParser;
-use tantivy::schema::*;
-use tantivy::snippet::{Snippet, SnippetGenerator};
-use tantivy::{doc, Index, IndexWriter};
-use tempfile::TempDir;
-
-fn main() -> tantivy::Result<()> {
- // Let's create a temporary directory for the
- // sake of this example
- let index_path = TempDir::new()?;
-
- // # Defining the schema
- let mut schema_builder = Schema::builder();
- let title = schema_builder.add_text_field("title", TEXT | STORED);
- let body = schema_builder.add_text_field("body", TEXT | STORED);
- let schema = schema_builder.build();
-
- // # Indexing documents
- let index = Index::create_in_dir(&index_path, schema)?;
-
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
-
- // we'll only need one doc for this example.
- index_writer.add_document(doc!(
- title => "Of Mice and Men",
- body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
- bank and runs deep and green. The water is warm too, for it has slipped twinkling \
- over the yellow sands in the sunlight before reaching the narrow pool. On one \
- side of the river the golden foothill slopes curve up to the strong and rocky \
- Gabilan Mountains, but on the valley side the water is lined with trees—willows \
- fresh and green with every spring, carrying in their lower leaf junctures the \
- debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
- limbs and branches that arch over the pool"
- ))?;
- // ...
- index_writer.commit()?;
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let query_parser = QueryParser::for_index(&index, vec![title, body]);
- let query = query_parser.parse_query("sycamore spring")?;
-
- let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
-
- let snippet_generator = SnippetGenerator::create(&searcher, &*query, body)?;
-
- for (score, doc_address) in top_docs {
- let doc = searcher.doc::<TantivyDocument>(doc_address)?;
- let snippet = snippet_generator.snippet_from_doc(&doc);
- println!("Document score {score}:");
- println!("title: {}", doc.get_first(title).unwrap().as_str().unwrap());
- println!("snippet: {}", snippet.to_html());
- println!("custom highlighting: {}", highlight(snippet));
- }
-
- Ok(())
-}
-
-fn highlight(snippet: Snippet) -> String {
- let mut result = String::new();
- let mut start_from = 0;
-
- for fragment_range in snippet.highlighted() {
- result.push_str(&snippet.fragment()[start_from..fragment_range.start]);
- result.push_str(" --> ");
- result.push_str(&snippet.fragment()[fragment_range.clone()]);
- result.push_str(" <-- ");
- start_from = fragment_range.end;
- }
-
- result.push_str(&snippet.fragment()[start_from..]);
- result
-}
diff --git a/crates/tantivy/examples/stop_words.rs b/crates/tantivy/examples/stop_words.rs
deleted file mode 100644
index 80dab3fe..00000000
--- a/crates/tantivy/examples/stop_words.rs
+++ /dev/null
@@ -1,114 +0,0 @@
-// # Stop Words Example
-//
-// This example covers the basic usage of stop words
-// with tantivy
-//
-// We will :
-// - define our schema
-// - create an index in a directory
-// - add a few stop words
-// - index few documents in our index
-
-// ---
-// Importing tantivy...
-use tantivy::collector::TopDocs;
-use tantivy::query::QueryParser;
-use tantivy::schema::*;
-use tantivy::tokenizer::*;
-use tantivy::{doc, Index, IndexWriter};
-
-fn main() -> tantivy::Result<()> {
- // this example assumes you understand the content in `basic_search`
- let mut schema_builder = Schema::builder();
-
- // This configures your custom options for how tantivy will
- // store and process your content in the index; The key
- // to note is that we are setting the tokenizer to `stoppy`
- // which will be defined and registered below.
- let text_field_indexing = TextFieldIndexing::default()
- .set_tokenizer("stoppy")
- .set_index_option(IndexRecordOption::WithFreqsAndPositions);
- let text_options = TextOptions::default()
- .set_indexing_options(text_field_indexing)
- .set_stored();
-
- // Our first field is title.
- schema_builder.add_text_field("title", text_options);
-
- // Our second field is body.
- let text_field_indexing = TextFieldIndexing::default()
- .set_tokenizer("stoppy")
- .set_index_option(IndexRecordOption::WithFreqsAndPositions);
- let text_options = TextOptions::default()
- .set_indexing_options(text_field_indexing)
- .set_stored();
- schema_builder.add_text_field("body", text_options);
-
- let schema = schema_builder.build();
-
- let index = Index::create_in_ram(schema.clone());
-
- // This tokenizer lowers all of the text (to help with stop word matching)
- // then removes all instances of `the` and `and` from the corpus
- let tokenizer = TextAnalyzer::builder(SimpleTokenizer::default())
- .filter(LowerCaser)
- .filter(StopWordFilter::remove(vec![
- "the".to_string(),
- "and".to_string(),
- ]))
- .build();
-
- index.tokenizers().register("stoppy", tokenizer);
-
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
-
- let title = schema.get_field("title").unwrap();
- let body = schema.get_field("body").unwrap();
-
- index_writer.add_document(doc!(
- title => "The Old Man and the Sea",
- body => "He was an old man who fished alone in a skiff in the Gulf Stream and \
- he had gone eighty-four days now without taking a fish."
- ))?;
-
- index_writer.add_document(doc!(
- title => "Of Mice and Men",
- body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
- bank and runs deep and green. The water is warm too, for it has slipped twinkling \
- over the yellow sands in the sunlight before reaching the narrow pool. On one \
- side of the river the golden foothill slopes curve up to the strong and rocky \
- Gabilan Mountains, but on the valley side the water is lined with trees—willows \
- fresh and green with every spring, carrying in their lower leaf junctures the \
- debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
- limbs and branches that arch over the pool"
- ))?;
-
- index_writer.add_document(doc!(
- title => "Frankenstein",
- body => "You will rejoice to hear that no disaster has accompanied the commencement of an \
- enterprise which you have regarded with such evil forebodings. I arrived here \
- yesterday, and my first task is to assure my dear sister of my welfare and \
- increasing confidence in the success of my undertaking."
- ))?;
-
- index_writer.commit()?;
-
- let reader = index.reader()?;
-
- let searcher = reader.searcher();
-
- let query_parser = QueryParser::for_index(&index, vec![title, body]);
-
- // stop words are applied on the query as well.
- // The following will be equivalent to `title:frankenstein`
- let query = query_parser.parse_query("title:\"the Frankenstein\"")?;
- let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
-
- for (score, doc_address) in top_docs {
- let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
- println!("\n==\nDocument score {score}:");
- println!("{}", retrieved_doc.to_json(&schema));
- }
-
- Ok(())
-}
diff --git a/crates/tantivy/examples/warmer.rs b/crates/tantivy/examples/warmer.rs
deleted file mode 100644
index 1cae9d34..00000000
--- a/crates/tantivy/examples/warmer.rs
+++ /dev/null
@@ -1,225 +0,0 @@
-use std::cmp::Reverse;
-use std::collections::{HashMap, HashSet};
-use std::sync::{Arc, RwLock, Weak};
-
-use tantivy::collector::TopDocs;
-use tantivy::index::SegmentId;
-use tantivy::query::QueryParser;
-use tantivy::schema::{Schema, FAST, TEXT};
-use tantivy::{
- doc, DocAddress, DocId, Index, IndexWriter, Opstamp, Searcher, SearcherGeneration,
- SegmentReader, Warmer,
-};
-
-// This example shows how warmers can be used to
-// load values from an external sources and
-// tie their lifecycle to that of the index segments
-// using the Warmer API.
-//
-// In this example, we assume an e-commerce search engine.
-
-type ProductId = u64;
-
-type Price = u32;
-
-pub trait PriceFetcher: Send + Sync + 'static {
- fn fetch_prices(&self, product_ids: &[ProductId]) -> Vec<Price>;
-}
-
-type SegmentKey = (SegmentId, Option<Opstamp>);
-
-struct DynamicPriceColumn {
- field: String,
- price_cache: RwLock<HashMap<SegmentKey, Arc<Vec<Price>>>>,
- price_fetcher: Box<dyn PriceFetcher>,
-}
-
-impl DynamicPriceColumn {
- pub fn with_product_id_field<T: PriceFetcher>(field: String, price_fetcher: T) -> Self {
- DynamicPriceColumn {
- field,
- price_cache: Default::default(),
- price_fetcher: Box::new(price_fetcher),
- }
- }
-
- pub fn price_for_segment(&self, segment_reader: &SegmentReader) -> Option<Arc<Vec<Price>>> {
- let segment_key = (segment_reader.segment_id(), segment_reader.delete_opstamp());
- self.price_cache.read().unwrap().get(&segment_key).cloned()
- }
-}
-impl Warmer for DynamicPriceColumn {
- fn warm(&self, searcher: &Searcher) -> tantivy::Result<()> {
- for segment in searcher.segment_readers() {
- let product_id_reader = segment
- .fast_fields()
- .u64(&self.field)?
- .first_or_default_col(0);
- let product_ids: Vec<ProductId> = segment
- .doc_ids_alive()
- .map(|doc| product_id_reader.get_val(doc))
- .collect();
-
- let mut prices = self.price_fetcher.fetch_prices(&product_ids).into_iter();
-
- let prices: Vec<Price> = (0..segment.max_doc())
- .map(|doc| {
- if !segment.is_deleted(doc) {
- prices.next().unwrap()
- } else {
- 0
- }
- })
- .collect();
-
- let key = (segment.segment_id(), segment.delete_opstamp());
- self.price_cache
- .write()
- .unwrap()
- .insert(key, Arc::new(prices));
- }
-
- Ok(())
- }
-
- fn garbage_collect(&self, live_generations: &[&SearcherGeneration]) {
- let live_keys: HashSet<SegmentKey> = live_generations
- .iter()
- .flat_map(|gen| gen.segments())
- .map(|(&segment_id, &opstamp)| (segment_id, opstamp))
- .collect();
-
- self.price_cache
- .write()
- .unwrap()
- .retain(|key, _| live_keys.contains(key));
- }
-}
-
-// For the sake of this example, the table is just an editable HashMap behind a RwLock.
-// This map represents a map (ProductId -> Price)
-//
-// In practise, it could be fetching things from an external service, like a SQL table.
-#[derive(Default, Clone)]
-pub struct ExternalPriceTable {
- prices: Arc<RwLock<HashMap<ProductId, Price>>>,
-}
-
-impl ExternalPriceTable {
- pub fn update_price(&self, product_id: ProductId, price: Price) {
- self.prices.write().unwrap().insert(product_id, price);
- }
-}
-
-impl PriceFetcher for ExternalPriceTable {
- fn fetch_prices(&self, product_ids: &[ProductId]) -> Vec<Price> {
- let prices = self.prices.read().unwrap();
-
- product_ids
- .iter()
- .map(|product_id| prices.get(product_id).cloned().unwrap_or(0))
- .collect()
- }
-}
-
-fn main() -> tantivy::Result<()> {
- // Declaring our schema.
- let mut schema_builder = Schema::builder();
- // The product id is assumed to be a primary id for our external price source.
- let product_id = schema_builder.add_u64_field("product_id", FAST);
- let text = schema_builder.add_text_field("text", TEXT);
- let schema: Schema = schema_builder.build();
-
- let price_table = ExternalPriceTable::default();
- let price_dynamic_column = Arc::new(DynamicPriceColumn::with_product_id_field(
- "product_id".to_string(),
- price_table.clone(),
- ));
- price_table.update_price(OLIVE_OIL, 12);
- price_table.update_price(GLOVES, 13);
- price_table.update_price(SNEAKERS, 80);
-
- const OLIVE_OIL: ProductId = 323423;
- const GLOVES: ProductId = 3966623;
- const SNEAKERS: ProductId = 23222;
-
- let index = Index::create_in_ram(schema);
- let mut writer: IndexWriter = index.writer_with_num_threads(1, 15_000_000)?;
- writer.add_document(doc!(product_id=>OLIVE_OIL, text=>"cooking olive oil from greece"))?;
- writer.add_document(doc!(product_id=>GLOVES, text=>"kitchen gloves, perfect for cooking"))?;
- writer.add_document(doc!(product_id=>SNEAKERS, text=>"uber sweet sneakers"))?;
- writer.commit()?;
-
- let warmers = vec![Arc::downgrade(&price_dynamic_column) as Weak<dyn Warmer>];
- let reader = index.reader_builder().warmers(warmers).try_into()?;
-
- let query_parser = QueryParser::for_index(&index, vec![text]);
- let query = query_parser.parse_query("cooking")?;
-
- let searcher = reader.searcher();
- let score_by_price = move |segment_reader: &SegmentReader| {
- let price = price_dynamic_column
- .price_for_segment(segment_reader)
- .unwrap();
- move |doc_id: DocId| Reverse(price[doc_id as usize])
- };
-
- let most_expensive_first = TopDocs::with_limit(10).custom_score(score_by_price);
-
- let hits = searcher.search(&query, &most_expensive_first)?;
- assert_eq!(
- &hits,
- &[
- (
- Reverse(12u32),
- DocAddress {
- segment_ord: 0,
- doc_id: 0u32
- }
- ),
- (
- Reverse(13u32),
- DocAddress {
- segment_ord: 0,
- doc_id: 1u32
- }
- ),
- ]
- );
-
- // Olive oil just got more expensive!
- price_table.update_price(OLIVE_OIL, 15);
-
- // The price update are directly reflected on `reload`.
- //
- // Be careful here though!...
- // You may have spotted that we are still using the same `Searcher`.
- //
- // It is up to the `Warmer` implementer to decide how
- // to control this behavior.
-
- reader.reload()?;
-
- let hits_with_new_prices = searcher.search(&query, &most_expensive_first)?;
- assert_eq!(
- &hits_with_new_prices,
- &[
- (
- Reverse(13u32),
- DocAddress {
- segment_ord: 0,
- doc_id: 1u32
- }
- ),
- (
- Reverse(15u32),
- DocAddress {
- segment_ord: 0,
- doc_id: 0u32
- }
- ),
- ]
- );
-
- Ok(())
-}
diff --git a/crates/tantivy/src/collector/count_collector.rs b/crates/tantivy/src/collector/count_collector.rs
index dcd10224..60c73bd2 100644
--- a/crates/tantivy/src/collector/count_collector.rs
+++ b/crates/tantivy/src/collector/count_collector.rs
@@ -4,35 +4,6 @@ use crate::{DocId, Score, SegmentOrdinal, SegmentReader};
/// `CountCollector` collector only counts how many
/// documents match the query.
-///
-/// ```rust
-/// use tantivy::collector::Count;
-/// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index};
-///
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
-///
-/// let mut index_writer = index.writer(15_000_000).unwrap();
-/// index_writer.add_document(doc!(title => "The Name of the Wind")).unwrap();
-/// index_writer.add_document(doc!(title => "The Diary of Muadib")).unwrap();
-/// index_writer.add_document(doc!(title => "A Dairy Cow")).unwrap();
-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl")).unwrap();
-/// assert!(index_writer.commit().is_ok());
-///
-/// let reader = index.reader().unwrap();
-/// let searcher = reader.searcher();
-///
-/// // Here comes the important part
-/// let query_parser = QueryParser::for_index(&index, vec![title]);
-/// let query = query_parser.parse_query("diary").unwrap();
-/// let count = searcher.search(&query, &Count).unwrap();
-///
-/// assert_eq!(count, 2);
-/// ```
pub struct Count;
impl Collector for Count {
diff --git a/crates/tantivy/src/collector/filter_collector_wrapper.rs b/crates/tantivy/src/collector/filter_collector_wrapper.rs
index 259ec49d..df121222 100644
--- a/crates/tantivy/src/collector/filter_collector_wrapper.rs
+++ b/crates/tantivy/src/collector/filter_collector_wrapper.rs
@@ -23,46 +23,6 @@ use crate::{DocId, Score, SegmentReader};
/// - documents with no values are filtered out.
/// - documents with several values are accepted if at least one value matches the predicate.
///
-///
-/// ```rust
-/// use tantivy::collector::{TopDocs, FilterCollector};
-/// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT, FAST};
-/// use tantivy::{doc, DocAddress, Index};
-///
-/// # fn main() -> tantivy::Result<()> {
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let price = schema_builder.add_u64_field("price", FAST);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
-///
-/// let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
-/// index_writer.add_document(doc!(title => "The Name of the Wind", price => 30_200u64))?;
-/// index_writer.add_document(doc!(title => "The Diary of Muadib", price => 29_240u64))?;
-/// index_writer.add_document(doc!(title => "A Dairy Cow", price => 21_240u64))?;
-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl", price => 20_120u64))?;
-/// index_writer.commit()?;
-///
-/// let reader = index.reader()?;
-/// let searcher = reader.searcher();
-///
-/// let query_parser = QueryParser::for_index(&index, vec![title]);
-/// let query = query_parser.parse_query("diary")?;
-/// let no_filter_collector = FilterCollector::new("price".to_string(), |value: u64| value > 20_120u64, TopDocs::with_limit(2));
-/// let top_docs = searcher.search(&query, &no_filter_collector)?;
-///
-/// assert_eq!(top_docs.len(), 1);
-/// assert_eq!(top_docs[0].1, DocAddress::new(0, 1));
-///
-/// let filter_all_collector: FilterCollector<_, _, u64> = FilterCollector::new("price".to_string(), |value| value < 5u64, TopDocs::with_limit(2));
-/// let filtered_top_docs = searcher.search(&query, &filter_all_collector)?;
-///
-/// assert_eq!(filtered_top_docs.len(), 0);
-/// # Ok(())
-/// # }
-/// ```
-///
/// Note that this is limited to fast fields which implement the
/// [`FastValue`][crate::fastfield::FastValue] trait, e.g. `u64` but not `&[u8]`.
/// To filter based on a bytes fast field, use a [`BytesFilterCollector`] instead.
@@ -191,41 +151,6 @@ where
/// In other words,
/// - documents with no values are filtered out.
/// - documents with several values are accepted if at least one value matches the predicate.
-///
-/// ```rust
-/// use tantivy::collector::{TopDocs, BytesFilterCollector};
-/// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT, FAST};
-/// use tantivy::{doc, DocAddress, Index};
-///
-/// # fn main() -> tantivy::Result<()> {
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let barcode = schema_builder.add_bytes_field("barcode", FAST);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
-///
-/// let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
-/// index_writer.add_document(doc!(title => "The Name of the Wind", barcode => &b"010101"[..]))?;
-/// index_writer.add_document(doc!(title => "The Diary of Muadib", barcode => &b"110011"[..]))?;
-/// index_writer.add_document(doc!(title => "A Dairy Cow", barcode => &b"110111"[..]))?;
-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl", barcode => &b"011101"[..]))?;
-/// index_writer.add_document(doc!(title => "Bridget Jones's Diary"))?;
-/// index_writer.commit()?;
-///
-/// let reader = index.reader()?;
-/// let searcher = reader.searcher();
-///
-/// let query_parser = QueryParser::for_index(&index, vec![title]);
-/// let query = query_parser.parse_query("diary")?;
-/// let filter_collector = BytesFilterCollector::new("barcode".to_string(), |bytes: &[u8]| bytes.starts_with(b"01"), TopDocs::with_limit(2));
-/// let top_docs = searcher.search(&query, &filter_collector)?;
-///
-/// assert_eq!(top_docs.len(), 1);
-/// assert_eq!(top_docs[0].1, DocAddress::new(0, 3));
-/// # Ok(())
-/// # }
-/// ```
pub struct BytesFilterCollector<TCollector, TPredicate>
where
TPredicate: 'static + Clone,
diff --git a/crates/tantivy/src/collector/mod.rs b/crates/tantivy/src/collector/mod.rs
index b78e0207..1325f43e 100644
--- a/crates/tantivy/src/collector/mod.rs
+++ b/crates/tantivy/src/collector/mod.rs
@@ -33,35 +33,6 @@
//! The resulting `Fruit` will then be a typed tuple with each collector's original fruits
//! in their respective position.
//!
-//! ```rust
-//! # use tantivy::schema::*;
-//! # use tantivy::*;
-//! # use tantivy::query::*;
-//! use tantivy::collector::{Count, TopDocs};
-//! #
-//! # fn main() -> tantivy::Result<()> {
-//! # let mut schema_builder = Schema::builder();
-//! # let title = schema_builder.add_text_field("title", TEXT);
-//! # let schema = schema_builder.build();
-//! # let index = Index::create_in_ram(schema);
-//! # let mut index_writer = index.writer(15_000_000)?;
-//! # index_writer.add_document(doc!(
-//! # title => "The Name of the Wind",
-//! # ))?;
-//! # index_writer.add_document(doc!(
-//! # title => "The Diary of Muadib",
-//! # ))?;
-//! # index_writer.commit()?;
-//! # let reader = index.reader()?;
-//! # let searcher = reader.searcher();
-//! # let query_parser = QueryParser::for_index(&index, vec![title]);
-//! # let query = query_parser.parse_query("diary")?;
-//! let (doc_count, top_docs): (usize, Vec<(Score, DocAddress)>) =
-//! searcher.search(&query, &(Count, TopDocs::with_limit(2)))?;
-//! # Ok(())
-//! # }
-//! ```
-//!
//! The `Collector` trait is implemented for up to 4 collectors.
//! If you have more than 4 collectors, you can either group them into
//! tuples of tuples `(a,(b,(c,d)))`, or rely on [`MultiCollector`].
diff --git a/crates/tantivy/src/collector/multi_collector.rs b/crates/tantivy/src/collector/multi_collector.rs
index da4d222a..7acf745d 100644
--- a/crates/tantivy/src/collector/multi_collector.rs
+++ b/crates/tantivy/src/collector/multi_collector.rs
@@ -124,43 +124,6 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
/// in a tuple. See the
/// [Combining several collectors section of the collector
/// documentation](./index.html#combining-several-collectors).
-///
-/// ```rust
-/// use tantivy::collector::{Count, TopDocs, MultiCollector};
-/// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index};
-///
-/// # fn main() -> tantivy::Result<()> {
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
-/// let mut index_writer = index.writer(15_000_000)?;
-/// index_writer.add_document(doc!(title => "The Name of the Wind"))?;
-/// index_writer.add_document(doc!(title => "The Diary of Muadib"))?;
-/// index_writer.add_document(doc!(title => "A Dairy Cow"))?;
-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"))?;
-/// index_writer.commit()?;
-///
-/// let reader = index.reader()?;
-/// let searcher = reader.searcher();
-///
-/// let mut collectors = MultiCollector::new();
-/// let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));
-/// let count_handle = collectors.add_collector(Count);
-/// let query_parser = QueryParser::for_index(&index, vec![title]);
-/// let query = query_parser.parse_query("diary").unwrap();
-/// let mut multi_fruit = searcher.search(&query, &collectors).unwrap();
-///
-/// let count = count_handle.extract(&mut multi_fruit);
-/// let top_docs = top_docs_handle.extract(&mut multi_fruit);
-///
-/// assert_eq!(count, 2);
-/// assert_eq!(top_docs.len(), 2);
-/// # Ok(())
-/// # }
-/// ```
#[allow(clippy::type_complexity)]
#[derive(Default)]
pub struct MultiCollector<'a> {
diff --git a/crates/tantivy/src/collector/top_score_collector.rs b/crates/tantivy/src/collector/top_score_collector.rs
index 305050c7..a3eb05c1 100644
--- a/crates/tantivy/src/collector/top_score_collector.rs
+++ b/crates/tantivy/src/collector/top_score_collector.rs
@@ -96,38 +96,6 @@ where
/// document score, for stable sorting `PartialOrd` needs to resolve on other fields
/// like docid in case of score equality.
/// Only then, it is suitable for pagination.
-///
-/// ```rust
-/// use tantivy::collector::TopDocs;
-/// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, DocAddress, Index};
-///
-/// # fn main() -> tantivy::Result<()> {
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
-///
-/// let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
-/// index_writer.add_document(doc!(title => "The Name of the Wind"))?;
-/// index_writer.add_document(doc!(title => "The Diary of Muadib"))?;
-/// index_writer.add_document(doc!(title => "A Dairy Cow"))?;
-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"))?;
-/// index_writer.commit()?;
-///
-/// let reader = index.reader()?;
-/// let searcher = reader.searcher();
-///
-/// let query_parser = QueryParser::for_index(&index, vec![title]);
-/// let query = query_parser.parse_query("diary")?;
-/// let top_docs = searcher.search(&query, &TopDocs::with_limit(2))?;
-///
-/// assert_eq!(top_docs[0].1, DocAddress::new(0, 1));
-/// assert_eq!(top_docs[1].1, DocAddress::new(0, 3));
-/// # Ok(())
-/// # }
-/// ```
pub struct TopDocs(TopCollector<Score>);
impl fmt::Debug for TopDocs {
@@ -201,40 +169,6 @@ impl TopDocs {
/// Lucene's TopDocsCollector.
///
/// # Example
- ///
- /// ```rust
- /// use tantivy::collector::TopDocs;
- /// use tantivy::query::QueryParser;
- /// use tantivy::schema::{Schema, TEXT};
- /// use tantivy::{doc, DocAddress, Index};
- ///
- /// # fn main() -> tantivy::Result<()> {
- /// let mut schema_builder = Schema::builder();
- /// let title = schema_builder.add_text_field("title", TEXT);
- /// let schema = schema_builder.build();
- /// let index = Index::create_in_ram(schema);
- ///
- /// let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
- /// index_writer.add_document(doc!(title => "The Name of the Wind"))?;
- /// index_writer.add_document(doc!(title => "The Diary of Muadib"))?;
- /// index_writer.add_document(doc!(title => "A Dairy Cow"))?;
- /// index_writer.add_document(doc!(title => "The Diary of a Young Girl"))?;
- /// index_writer.add_document(doc!(title => "The Diary of Lena Mukhina"))?;
- /// index_writer.commit()?;
- ///
- /// let reader = index.reader()?;
- /// let searcher = reader.searcher();
- ///
- /// let query_parser = QueryParser::for_index(&index, vec![title]);
- /// let query = query_parser.parse_query("diary")?;
- /// let top_docs = searcher.search(&query, &TopDocs::with_limit(2).and_offset(1))?;
- ///
- /// assert_eq!(top_docs.len(), 2);
- /// assert_eq!(top_docs[0].1, DocAddress::new(0, 4));
- /// assert_eq!(top_docs[1].1, DocAddress::new(0, 3));
- /// Ok(())
- /// # }
- /// ```
#[must_use]
pub fn and_offset(self, offset: usize) -> TopDocs {
TopDocs(self.0.and_offset(offset))
@@ -249,64 +183,6 @@ impl TopDocs {
/// will return returns a monotonic u64-representation (ie. the order is still correct) of
/// the requested field type.
///
- /// # Example
- ///
- /// ```rust
- /// # use tantivy::schema::{Schema, FAST, TEXT};
- /// # use tantivy::{doc, Index, DocAddress, Order};
- /// # use tantivy::query::{Query, QueryParser};
- /// use tantivy::Searcher;
- /// use tantivy::collector::TopDocs;
- ///
- /// # fn main() -> tantivy::Result<()> {
- /// # let mut schema_builder = Schema::builder();
- /// # let title = schema_builder.add_text_field("title", TEXT);
- /// # let rating = schema_builder.add_u64_field("rating", FAST);
- /// # let schema = schema_builder.build();
- /// #
- /// # let index = Index::create_in_ram(schema);
- /// # let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
- /// # index_writer.add_document(doc!(title => "The Name of the Wind", rating => 92u64))?;
- /// # index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64))?;
- /// # index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64))?;
- /// # index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64))?;
- /// # index_writer.commit()?;
- /// # let reader = index.reader()?;
- /// # let query = QueryParser::for_index(&index, vec![title]).parse_query("diary")?;
- /// # let top_docs = docs_sorted_by_rating(&reader.searcher(), &query)?;
- /// # assert_eq!(top_docs,
- /// # vec![(97u64, DocAddress::new(0u32, 1)),
- /// # (80u64, DocAddress::new(0u32, 3))]);
- /// # Ok(())
- /// # }
- /// /// Searches the document matching the given query, and
- /// /// collects the top 10 documents, order by the u64-`field`
- /// /// given in argument.
- /// fn docs_sorted_by_rating(searcher: &Searcher,
- /// query: &dyn Query)
- /// -> tantivy::Result<Vec<(u64, DocAddress)>> {
- ///
- /// // This is where we build our topdocs collector
- /// //
- /// // Note the `rating_field` needs to be a FAST field here.
- /// let top_books_by_rating = TopDocs
- /// ::with_limit(10)
- /// .order_by_fast_field("rating", Order::Desc);
- ///
- /// // ... and here are our documents. Note this is a simple vec.
- /// // The `u64` in the pair is the value of our fast field for
- /// // each documents.
- /// //
- /// // The vec is sorted decreasingly by `sort_by_field`, and has a
- /// // length of 10, or less if not enough documents matched the
- /// // query.
- /// let resulting_docs: Vec<(u64, DocAddress)> =
- /// searcher.search(query, &top_books_by_rating)?;
- ///
- /// Ok(resulting_docs)
- /// }
- /// ```
- ///
/// # See also
///
/// To comfortably work with `u64`s, `i64`s, `f64`s, or `date`s, please refer to
@@ -435,76 +311,6 @@ impl TopDocs {
/// In more serious application, this tweaking could involve running a
/// learning-to-rank model over various features
///
- /// ```rust
- /// # use tantivy::schema::{Schema, FAST, TEXT};
- /// # use tantivy::{doc, Index, DocAddress, DocId, Score};
- /// # use tantivy::query::QueryParser;
- /// use tantivy::SegmentReader;
- /// use tantivy::collector::TopDocs;
- /// use tantivy::schema::Field;
- ///
- /// fn create_schema() -> Schema {
- /// let mut schema_builder = Schema::builder();
- /// schema_builder.add_text_field("product_name", TEXT);
- /// schema_builder.add_u64_field("popularity", FAST);
- /// schema_builder.build()
- /// }
- ///
- /// fn create_index() -> tantivy::Result<Index> {
- /// let schema = create_schema();
- /// let index = Index::create_in_ram(schema);
- /// let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
- /// let product_name = index.schema().get_field("product_name").unwrap();
- /// let popularity: Field = index.schema().get_field("popularity").unwrap();
- /// index_writer.add_document(doc!(product_name => "The Diary of Muadib", popularity => 1u64))?;
- /// index_writer.add_document(doc!(product_name => "A Dairy Cow", popularity => 10u64))?;
- /// index_writer.add_document(doc!(product_name => "The Diary of a Young Girl", popularity => 15u64))?;
- /// index_writer.commit()?;
- /// Ok(index)
- /// }
- ///
- /// let index = create_index().unwrap();
- /// let product_name = index.schema().get_field("product_name").unwrap();
- /// let popularity: Field = index.schema().get_field("popularity").unwrap();
- ///
- /// let user_query_str = "diary";
- /// let query_parser = QueryParser::for_index(&index, vec![product_name]);
- /// let query = query_parser.parse_query(user_query_str).unwrap();
- ///
- /// // This is where we build our collector with our custom score.
- /// let top_docs_by_custom_score = TopDocs
- /// ::with_limit(10)
- /// .tweak_score(move |segment_reader: &SegmentReader| {
- /// // The argument is a function that returns our scoring
- /// // function.
- /// //
- /// // The point of this "mother" function is to gather all
- /// // of the segment level information we need for scoring.
- /// // Typically, fast_fields.
- /// //
- /// // In our case, we will get a reader for the popularity
- /// // fast field. For simplicity we read the first or default value in the fast
- /// // field.
- /// let popularity_reader =
- /// segment_reader.fast_fields().u64("popularity").unwrap().first_or_default_col(0);
- ///
- /// // We can now define our actual scoring function
- /// move |doc: DocId, original_score: Score| {
- /// let popularity: u64 = popularity_reader.get_val(doc);
- /// // Well.. For the sake of the example we use a simple logarithm
- /// // function.
- /// let popularity_boost_score = ((2u64 + popularity) as Score).log2();
- /// popularity_boost_score * original_score
- /// }
- /// });
- /// let reader = index.reader().unwrap();
- /// let searcher = reader.searcher();
- /// // ... and here are our documents. Note this is a simple vec.
- /// // The `Score` in the pair is our tweaked score.
- /// let resulting_docs: Vec<(Score, DocAddress)> =
- /// searcher.search(&query, &top_docs_by_custom_score).unwrap();
- /// ```
- ///
/// # See also
/// - [custom_score(...)](TopDocs::custom_score)
pub fn tweak_score<TScore, TScoreSegmentTweaker, TScoreTweaker>(
@@ -539,85 +345,6 @@ impl TopDocs {
/// It can be used if your search engine relies on a learning-to-rank model for instance,
/// which does not rely on the term frequencies or positions as features.
///
- /// # Example
- ///
- /// ```rust
- /// # use tantivy::schema::{Schema, FAST, TEXT};
- /// # use tantivy::{doc, Index, DocAddress, DocId};
- /// # use tantivy::query::QueryParser;
- /// use tantivy::SegmentReader;
- /// use tantivy::collector::TopDocs;
- /// use tantivy::schema::Field;
- ///
- /// # fn create_schema() -> Schema {
- /// # let mut schema_builder = Schema::builder();
- /// # schema_builder.add_text_field("product_name", TEXT);
- /// # schema_builder.add_u64_field("popularity", FAST);
- /// # schema_builder.add_u64_field("boosted", FAST);
- /// # schema_builder.build()
- /// # }
- /// #
- /// # fn main() -> tantivy::Result<()> {
- /// # let schema = create_schema();
- /// # let index = Index::create_in_ram(schema);
- /// # let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
- /// # let product_name = index.schema().get_field("product_name").unwrap();
- /// #
- /// let popularity: Field = index.schema().get_field("popularity").unwrap();
- /// let boosted: Field = index.schema().get_field("boosted").unwrap();
- /// # index_writer.add_document(doc!(boosted=>1u64, product_name => "The Diary of Muadib", popularity => 1u64))?;
- /// # index_writer.add_document(doc!(boosted=>0u64, product_name => "A Dairy Cow", popularity => 10u64))?;
- /// # index_writer.add_document(doc!(boosted=>0u64, product_name => "The Diary of a Young Girl", popularity => 15u64))?;
- /// # index_writer.commit()?;
- /// // ...
- /// # let user_query = "diary";
- /// # let query = QueryParser::for_index(&index, vec![product_name]).parse_query(user_query)?;
- ///
- /// // This is where we build our collector with our custom score.
- /// let top_docs_by_custom_score = TopDocs
- /// ::with_limit(10)
- /// .custom_score(move |segment_reader: &SegmentReader| {
- /// // The argument is a function that returns our scoring
- /// // function.
- /// //
- /// // The point of this "mother" function is to gather all
- /// // of the segment level information we need for scoring.
- /// // Typically, fast_fields.
- /// //
- /// // In our case, we will get a reader for the popularity
- /// // fast field and a boosted field.
- /// //
- /// // We want to get boosted items score, and when we get
- /// // a tie, return the item with the highest popularity.
- /// //
- /// // Note that this is implemented by using a `(u64, u64)`
- /// // as a score.
- /// let popularity_reader =
- /// segment_reader.fast_fields().u64("popularity").unwrap().first_or_default_col(0);
- /// let boosted_reader =
- /// segment_reader.fast_fields().u64("boosted").unwrap().first_or_default_col(0);
- ///
- /// // We can now define our actual scoring function
- /// move |doc: DocId| {
- /// let popularity: u64 = popularity_reader.get_val(doc);
- /// let boosted: u64 = boosted_reader.get_val(doc);
- /// // Score do not have to be `f64` in tantivy.
- /// // Here we return a couple to get lexicographical order
- /// // for free.
- /// (boosted, popularity)
- /// }
- /// });
- /// # let reader = index.reader()?;
- /// # let searcher = reader.searcher();
- /// // ... and here are our documents. Note this is a simple vec.
- /// // The `Score` in the pair is our tweaked score.
- /// let resulting_docs: Vec<((u64, u64), DocAddress)> =
- /// searcher.search(&*query, &top_docs_by_custom_score)?;
- ///
- /// # Ok(())
- /// # }
- /// ```
- ///
/// # See also
/// - [tweak_score(...)](TopDocs::tweak_score)
pub fn custom_score<TScore, TCustomSegmentScorer, TCustomScorer>(
diff --git a/crates/tantivy/src/error.rs b/crates/tantivy/src/error.rs
index ade94d54..6eeca6ec 100644
--- a/crates/tantivy/src/error.rs
+++ b/crates/tantivy/src/error.rs
@@ -10,8 +10,11 @@ use crate::directory::error::{
Incompatibility, LockError, OpenDirectoryError, OpenReadError, OpenWriteError,
};
use crate::fastfield::FastFieldNotAvailableError;
+use crate::schema;
use crate::schema::document::DeserializeError;
-use crate::{query, schema};
+
+#[cfg(test)]
+use crate::query;
/// Represents a `DataCorruption` error.
///
@@ -129,6 +132,7 @@ impl From<LockError> for TantivyError {
}
}
+#[cfg(test)]
impl From<query::QueryParserError> for TantivyError {
fn from(parsing_error: query::QueryParserError) -> TantivyError {
TantivyError::InvalidArgument(format!("Query is invalid. {parsing_error:?}"))
diff --git a/crates/tantivy/src/indexer/index_writer.rs b/crates/tantivy/src/indexer/index_writer.rs
index 5b96227b..a6c3924c 100644
--- a/crates/tantivy/src/indexer/index_writer.rs
+++ b/crates/tantivy/src/indexer/index_writer.rs
@@ -467,37 +467,6 @@ impl<D: Document> IndexWriter<D> {
/// Requires `commit`ing
/// Enables users to rebuild the index,
/// by clearing and resubmitting necessary documents
- ///
- /// ```rust
- /// use tantivy::collector::TopDocs;
- /// use tantivy::query::QueryParser;
- /// use tantivy::schema::*;
- /// use tantivy::{doc, Index};
- ///
- /// fn main() -> tantivy::Result<()> {
- /// let mut schema_builder = Schema::builder();
- /// let title = schema_builder.add_text_field("title", TEXT | STORED);
- /// let schema = schema_builder.build();
- ///
- /// let index = Index::create_in_ram(schema.clone());
- ///
- /// let mut index_writer = index.writer_with_num_threads(1, 50_000_000)?;
- /// index_writer.add_document(doc!(title => "The modern Promotheus"))?;
- /// index_writer.commit()?;
- ///
- /// let clear_res = index_writer.delete_all_documents().unwrap();
- /// // have to commit, otherwise deleted terms remain available
- /// index_writer.commit()?;
- ///
- /// let searcher = index.reader()?.searcher();
- /// let query_parser = QueryParser::for_index(&index, vec![title]);
- /// let query_promo = query_parser.parse_query("Promotheus")?;
- /// let top_docs_promo = searcher.search(&query_promo, &TopDocs::with_limit(1))?;
- ///
- /// assert!(top_docs_promo.is_empty());
- /// Ok(())
- /// }
- /// ```
pub fn delete_all_documents(&self) -> crate::Result<Opstamp> {
// Delete segments
self.segment_updater.remove_all_segments();
diff --git a/crates/tantivy/src/lib.rs b/crates/tantivy/src/lib.rs
index 14e52c50..394d95f4 100644
--- a/crates/tantivy/src/lib.rs
+++ b/crates/tantivy/src/lib.rs
@@ -13,90 +13,6 @@
//! Tantivy is a search engine library.
//! Think `Lucene`, but in Rust.
//!
-//! ```rust
-//! # use std::path::Path;
-//! # use tempfile::TempDir;
-//! # use tantivy::collector::TopDocs;
-//! # use tantivy::query::QueryParser;
-//! # use tantivy::schema::*;
-//! # use tantivy::{doc, DocAddress, Index, IndexWriter, Score};
-//! #
-//! # fn main() {
-//! # // Let's create a temporary directory for the
-//! # // sake of this example
-//! # if let Ok(dir) = TempDir::new() {
-//! # run_example(dir.path()).unwrap();
-//! # dir.close().unwrap();
-//! # }
-//! # }
-//! #
-//! # fn run_example(index_path: &Path) -> tantivy::Result<()> {
-//! // First we need to define a schema ...
-//!
-//! // `TEXT` means the field should be tokenized and indexed,
-//! // along with its term frequency and term positions.
-//! //
-//! // `STORED` means that the field will also be saved
-//! // in a compressed, row-oriented key-value store.
-//! // This store is useful to reconstruct the
-//! // documents that were selected during the search phase.
-//! let mut schema_builder = Schema::builder();
-//! let title = schema_builder.add_text_field("title", TEXT | STORED);
-//! let body = schema_builder.add_text_field("body", TEXT);
-//! let schema = schema_builder.build();
-//!
-//! // Indexing documents
-//!
-//! let index = Index::create_in_dir(index_path, schema.clone())?;
-//!
-//! // Here we use a buffer of 100MB that will be split
-//! // between indexing threads.
-//! let mut index_writer: IndexWriter = index.writer(100_000_000)?;
-//!
-//! // Let's index one documents!
-//! index_writer.add_document(doc!(
-//! title => "The Old Man and the Sea",
-//! body => "He was an old man who fished alone in a skiff in \
-//! the Gulf Stream and he had gone eighty-four days \
-//! now without taking a fish."
-//! ))?;
-//!
-//! // We need to call .commit() explicitly to force the
-//! // index_writer to finish processing the documents in the queue,
-//! // flush the current index to the disk, and advertise
-//! // the existence of new documents.
-//! index_writer.commit()?;
-//!
-//! // # Searching
-//!
-//! let reader = index.reader()?;
-//!
-//! let searcher = reader.searcher();
-//!
-//! let query_parser = QueryParser::for_index(&index, vec![title, body]);
-//!
-//! // QueryParser may fail if the query is not in the right
-//! // format. For user facing applications, this can be a problem.
-//! // A ticket has been opened regarding this problem.
-//! let query = query_parser.parse_query("sea whale")?;
-//!
-//! // Perform search.
-//! // `topdocs` contains the 10 most relevant doc ids, sorted by decreasing scores...
-//! let top_docs: Vec<(Score, DocAddress)> =
-//! searcher.search(&query, &TopDocs::with_limit(10))?;
-//!
-//! for (_score, doc_address) in top_docs {
-//! // Retrieve the actual content of documents given its `doc_address`.
-//! let retrieved_doc = searcher.doc::<TantivyDocument>(doc_address)?;
-//! println!("{}", retrieved_doc.to_json(&schema));
-//! }
-//!
-//! # Ok(())
-//! # }
-//! ```
-//!
-//!
-//!
//! A good place for you to get started is to check out
//! the example code (
//! [literate programming](https://tantivy-search.github.io/examples/basic_search.html) /
diff --git a/crates/tantivy/src/query/mod.rs b/crates/tantivy/src/query/mod.rs
index abe84d85..4083627a 100644
--- a/crates/tantivy/src/query/mod.rs
+++ b/crates/tantivy/src/query/mod.rs
@@ -16,7 +16,6 @@ mod more_like_this;
mod phrase_prefix_query;
mod phrase_query;
mod query;
-mod query_parser;
mod range_query;
mod reqopt_scorer;
mod scorer;
@@ -28,6 +27,9 @@ mod weight;
#[cfg(test)]
mod vec_docset;
+#[cfg(test)]
+mod query_parser;
+
pub mod grammar;
pub(crate) mod score_combiner;
pub use grammar::Occur;
@@ -52,6 +54,7 @@ pub use self::more_like_this::{MoreLikeThisQuery, MoreLikeThisQueryBuilder};
pub use self::phrase_prefix_query::PhrasePrefixQuery;
pub use self::phrase_query::PhraseQuery;
pub use self::query::{EnableScoring, Query, QueryClone};
+#[cfg(test)]
pub use self::query_parser::{QueryParser, QueryParserError};
pub use self::range_query::{FastFieldRangeWeight, IPFastFieldRangeWeight, RangeQuery};
pub use self::reqopt_scorer::RequiredOptionalScorer;
diff --git a/crates/tantivy/src/snippet/mod.rs b/crates/tantivy/src/snippet/mod.rs
index 16edac04..16d54953 100644
--- a/crates/tantivy/src/snippet/mod.rs
+++ b/crates/tantivy/src/snippet/mod.rs
@@ -8,46 +8,6 @@
//! It scans a document and constructs a snippet, which consists of sections where the search terms
//! have been found, stitched together with "..." in between sections if necessary.
//!
-//! ## Example
-//!
-//! ```rust
-//! # use tantivy::query::QueryParser;
-//! # use tantivy::schema::{Schema, TEXT};
-//! # use tantivy::{doc, Index};
-//! use tantivy::snippet::SnippetGenerator;
-//!
-//! # fn main() -> tantivy::Result<()> {
-//! # let mut schema_builder = Schema::builder();
-//! # let text_field = schema_builder.add_text_field("text", TEXT);
-//! # let schema = schema_builder.build();
-//! # let index = Index::create_in_ram(schema);
-//! # let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
-//! # let doc = doc!(text_field => r#"Comme je descendais des Fleuves impassibles,
-//! # Je ne me sentis plus guidé par les haleurs :
-//! # Des Peaux-Rouges criards les avaient pris pour cibles,
-//! # Les ayant cloués nus aux poteaux de couleurs.
-//! #
-//! # J'étais insoucieux de tous les équipages,
-//! # Porteur de blés flamands ou de cotons anglais.
-//! # Quand avec mes haleurs ont fini ces tapages,
-//! # Les Fleuves m'ont laissé descendre où je voulais.
-//! # "#);
-//! # index_writer.add_document(doc.clone())?;
-//! # index_writer.commit()?;
-//! # let query_parser = QueryParser::for_index(&index, vec![text_field]);
-//! // ...
-//! let query = query_parser.parse_query("haleurs flamands").unwrap();
-//! # let reader = index.reader()?;
-//! # let searcher = reader.searcher();
-//! let mut snippet_generator = SnippetGenerator::create(&searcher, &*query, text_field)?;
-//! snippet_generator.set_max_num_chars(100);
-//! let snippet = snippet_generator.snippet_from_doc(&doc);
-//! let snippet_html: String = snippet.to_html();
-//! assert_eq!(snippet_html, "Comme je descendais des Fleuves impassibles,\n Je ne me sentis plus guidé par les <b>haleurs</b> :\n Des");
-//! # Ok(())
-//! # }
-//! ```
-//!
//! You can also specify the maximum number of characters for the snippets generated with the
//! `set_max_num_chars` method. By default, this limit is set to 150.
//!
@@ -304,47 +264,6 @@ fn is_sorted(mut it: impl Iterator<Item = usize>) -> bool {
true
}
-/// `SnippetGenerator`
-///
-/// # Example
-///
-/// ```rust
-/// # use tantivy::query::QueryParser;
-/// # use tantivy::schema::{Schema, TEXT};
-/// # use tantivy::{doc, Index};
-/// use tantivy::snippet::SnippetGenerator;
-///
-/// # fn main() -> tantivy::Result<()> {
-/// # let mut schema_builder = Schema::builder();
-/// # let text_field = schema_builder.add_text_field("text", TEXT);
-/// # let schema = schema_builder.build();
-/// # let index = Index::create_in_ram(schema);
-/// # let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
-/// # let doc = doc!(text_field => r#"Comme je descendais des Fleuves impassibles,
-/// # Je ne me sentis plus guidé par les haleurs :
-/// # Des Peaux-Rouges criards les avaient pris pour cibles,
-/// # Les ayant cloués nus aux poteaux de couleurs.
-/// #
-/// # J'étais insoucieux de tous les équipages,
-/// # Porteur de blés flamands ou de cotons anglais.
-/// # Quand avec mes haleurs ont fini ces tapages,
-/// # Les Fleuves m'ont laissé descendre où je voulais.
-/// # "#);
-/// # index_writer.add_document(doc.clone())?;
-/// # index_writer.commit()?;
-/// # let query_parser = QueryParser::for_index(&index, vec![text_field]);
-/// // ...
-/// let query = query_parser.parse_query("haleurs flamands").unwrap();
-/// # let reader = index.reader()?;
-/// # let searcher = reader.searcher();
-/// let mut snippet_generator = SnippetGenerator::create(&searcher, &*query, text_field)?;
-/// snippet_generator.set_max_num_chars(100);
-/// let snippet = snippet_generator.snippet_from_doc(&doc);
-/// let snippet_html: String = snippet.to_html();
-/// assert_eq!(snippet_html, "Comme je descendais des Fleuves impassibles,\n Je ne me sentis plus guidé par les <b>haleurs</b> :\n Des");
-/// # Ok(())
-/// # }
-/// ```
pub struct SnippetGenerator {
terms_text: BTreeMap<String, Score>,
tokenizer: TextAnalyzer,