web search done right
mark the query parser in tantivy with #[cfg(test)] to ensure we don't accidentally use it instead of stracts
Mikkel Denker8af21448988b
diff --git a/crates/tantivy/examples/basic_search.rs b/crates/tantivy/examples/basic_search.rsdeleted file mode 100644index 3c9c6ca8..00000000--- a/crates/tantivy/examples/basic_search.rs+++ /dev/null@@ -1,239 +0,0 @@-// # Basic Example-//-// This example covers the basic functionalities of-// tantivy.-//-// We will :-// - define our schema-// - create an index in a directory-// - index a few documents into our index-// - search for the best document matching a basic query-// - retrieve the best document's original content.--// ----// Importing tantivy...-use tantivy::collector::TopDocs;-use tantivy::query::QueryParser;-use tantivy::schema::*;-use tantivy::{doc, Index, IndexWriter, ReloadPolicy};-use tempfile::TempDir;--fn main() -> tantivy::Result<()> {- // Let's create a temporary directory for the- // sake of this example- let index_path = TempDir::new()?;-- // # Defining the schema- //- // The Tantivy index requires a very strict schema.- // The schema declares which fields are in the index,- // and for each field, its type and "the way it should- // be indexed".-- // First we need to define a schema ...- let mut schema_builder = Schema::builder();-- // Our first field is title.- // We want full-text search for it, and we also want- // to be able to retrieve the document after the search.- //- // `TEXT | STORED` is some syntactic sugar to describe- // that.- //- // `TEXT` means the field should be tokenized and indexed,- // along with its term frequency and term positions.- //- // `STORED` means that the field will also be saved- // in a compressed, row-oriented key-value store.- // This store is useful for reconstructing the- // documents that were selected during the search phase.- schema_builder.add_text_field("title", TEXT | STORED);-- // Our second field is body.- // We want full-text search for it, but we do not- // need to be able to be able to retrieve it- // for our application.- //- // We can make our index lighter by omitting the `STORED` flag.- schema_builder.add_text_field("body", TEXT);-- let schema = schema_builder.build();-- // # Indexing documents- //- // Let's create a brand new index.- //- // This will actually just save a meta.json- // with our schema in the directory.- let index = Index::create_in_dir(&index_path, schema.clone())?;-- // To insert a document we will need an index writer.- // There must be only one writer at a time.- // This single `IndexWriter` is already- // multithreaded.- //- // Here we give tantivy a budget of `50MB`.- // Using a bigger memory_arena for the indexer may increase- // throughput, but 50 MB is already plenty.- let mut index_writer: IndexWriter = index.writer(50_000_000)?;-- // Let's index our documents!- // We first need a handle on the title and the body field.-- // ### Adding documents- //- // We can create a document manually, by setting the fields- // one by one in a Document object.- let title = schema.get_field("title").unwrap();- let body = schema.get_field("body").unwrap();-- let mut old_man_doc = TantivyDocument::default();- old_man_doc.add_text(title, "The Old Man and the Sea");- old_man_doc.add_text(- body,- "He was an old man who fished alone in a skiff in the Gulf Stream and he had gone \- eighty-four days now without taking a fish.",- );-- // ... and add it to the `IndexWriter`.- index_writer.add_document(old_man_doc)?;-- // For convenience, tantivy also comes with a macro to- // reduce the boilerplate above.- index_writer.add_document(doc!(- title => "Of Mice and Men",- body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \- bank and runs deep and green. The water is warm too, for it has slipped twinkling \- over the yellow sands in the sunlight before reaching the narrow pool. On one \- side of the river the golden foothill slopes curve up to the strong and rocky \- Gabilan Mountains, but on the valley side the water is lined with trees—willows \- fresh and green with every spring, carrying in their lower leaf junctures the \- debris of the winter’s flooding; and sycamores with mottled, white, recumbent \- limbs and branches that arch over the pool"- ))?;-- // Multivalued field just need to be repeated.- index_writer.add_document(doc!(- title => "Frankenstein",- title => "The Modern Prometheus",- body => "You will rejoice to hear that no disaster has accompanied the commencement of an \- enterprise which you have regarded with such evil forebodings. I arrived here \- yesterday, and my first task is to assure my dear sister of my welfare and \- increasing confidence in the success of my undertaking."- ))?;-- // This is an example, so we will only index 3 documents- // here. You can check out tantivy's tutorial to index- // the English wikipedia. Tantivy's indexing is rather fast.- // Indexing 5 million articles of the English wikipedia takes- // around 3 minutes on my computer!-- // ### Committing- //- // At this point our documents are not searchable.- //- //- // We need to call `.commit()` explicitly to force the- // `index_writer` to finish processing the documents in the queue,- // flush the current index to the disk, and advertise- // the existence of new documents.- //- // This call is blocking.- index_writer.commit()?;-- // If `.commit()` returns correctly, then all of the- // documents that have been added are guaranteed to be- // persistently indexed.- //- // In the scenario of a crash or a power failure,- // tantivy behaves as if it has rolled back to its last- // commit.-- // # Searching- //- // ### Searcher- //- // A reader is required first in order to search an index.- // It acts as a `Searcher` pool that reloads itself,- // depending on a `ReloadPolicy`.- //- // For a search server you will typically create one reader for the entire lifetime of your- // program, and acquire a new searcher for every single request.- //- // In the code below, we rely on the 'ON_COMMIT' policy: the reader- // will reload the index automatically after each commit.- let reader = index- .reader_builder()- .reload_policy(ReloadPolicy::OnCommitWithDelay)- .try_into()?;-- // We now need to acquire a searcher.- //- // A searcher points to a snapshotted, immutable version of the index.- //- // Some search experience might require more than- // one query. Using the same searcher ensures that all of these queries will run on the- // same version of the index.- //- // Acquiring a `searcher` is very cheap.- //- // You should acquire a searcher every time you start processing a request and- // and release it right after your query is finished.- let searcher = reader.searcher();-- // ### Query-- // The query parser can interpret human queries.- // Here, if the user does not specify which- // field they want to search, tantivy will search- // in both title and body.- let query_parser = QueryParser::for_index(&index, vec![title, body]);-- // `QueryParser` may fail if the query is not in the right- // format. For user facing applications, this can be a problem.- // A ticket has been opened regarding this problem.- let query = query_parser.parse_query("sea whale")?;-- // A query defines a set of documents, as- // well as the way they should be scored.- //- // A query created by the query parser is scored according- // to a metric called Tf-Idf, and will consider- // any document matching at least one of our terms.-- // ### Collectors- //- // We are not interested in all of the documents but- // only in the top 10. Keeping track of our top 10 best documents- // is the role of the `TopDocs` collector.-- // We can now perform our query.- let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;-- // The actual documents still need to be- // retrieved from Tantivy's store.- //- // Since the body field was not configured as stored,- // the document returned will only contain- // a title.- for (_score, doc_address) in top_docs {- let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;- println!("{}", retrieved_doc.to_json(&schema));- }-- // We can also get an explanation to understand- // how a found document got its score.- let query = query_parser.parse_query("title:sea^20 body:whale^70")?;-- let (_score, doc_address) = searcher- .search(&query, &TopDocs::with_limit(1))?- .into_iter()- .next()- .unwrap();-- let explanation = query.explain(&searcher, doc_address)?;-- println!("{}", explanation.to_pretty_json());-- Ok(())-}diff --git a/crates/tantivy/examples/custom_collector.rs b/crates/tantivy/examples/custom_collector.rsdeleted file mode 100644index 6c6e3475..00000000--- a/crates/tantivy/examples/custom_collector.rs+++ /dev/null@@ -1,186 +0,0 @@-// # Custom collector example-//-// This example shows how you can implement your own-// collector. As an example, we will compute a collector-// that computes the standard deviation of a given fast field.-//-// Of course, you can have a look at the tantivy's built-in collectors-// such as the `CountCollector` for more examples.--use tantivy::columnar::Column;-// ----// Importing tantivy...-use tantivy::collector::{Collector, SegmentCollector};-use tantivy::index::SegmentReader;-use tantivy::query::QueryParser;-use tantivy::schema::{Schema, FAST, INDEXED, TEXT};-use tantivy::{doc, Index, IndexWriter, Score};--#[derive(Default)]-struct Stats {- count: usize,- sum: f64,- squared_sum: f64,-}--impl Stats {- pub fn count(&self) -> usize {- self.count- }-- pub fn mean(&self) -> f64 {- self.sum / (self.count as f64)- }-- fn square_mean(&self) -> f64 {- self.squared_sum / (self.count as f64)- }-- pub fn standard_deviation(&self) -> f64 {- let mean = self.mean();- (self.square_mean() - mean * mean).sqrt()- }-- fn non_zero_count(self) -> Option<Stats> {- if self.count == 0 {- None- } else {- Some(self)- }- }-}--struct StatsCollector {- field: String,-}--impl StatsCollector {- fn with_field(field: String) -> StatsCollector {- StatsCollector { field }- }-}--impl Collector for StatsCollector {- // That's the type of our result.- // Our standard deviation will be a float.- type Fruit = Option<Stats>;-- type Child = StatsSegmentCollector;-- fn for_segment(- &self,- _segment_local_id: u32,- segment_reader: &SegmentReader,- ) -> tantivy::Result<StatsSegmentCollector> {- let fast_field_reader = segment_reader.fast_fields().u64(&self.field)?;- Ok(StatsSegmentCollector {- fast_field_reader,- stats: Stats::default(),- })- }-- fn requires_scoring(&self) -> bool {- // this collector does not care about score.- false- }-- fn merge_fruits(&self, segment_stats: Vec<Option<Stats>>) -> tantivy::Result<Option<Stats>> {- let mut stats = Stats::default();- for segment_stats in segment_stats.into_iter().flatten() {- stats.count += segment_stats.count;- stats.sum += segment_stats.sum;- stats.squared_sum += segment_stats.squared_sum;- }- Ok(stats.non_zero_count())- }-}--struct StatsSegmentCollector {- fast_field_reader: Column,- stats: Stats,-}--impl SegmentCollector for StatsSegmentCollector {- type Fruit = Option<Stats>;-- fn collect(&mut self, doc: u32, _score: Score) {- // Since we know the values are single value, we could call `first_or_default_col` on the- // column and fetch single values.- for value in self.fast_field_reader.values_for_doc(doc) {- let value = value as f64;- self.stats.count += 1;- self.stats.sum += value;- self.stats.squared_sum += value * value;- }- }-- fn harvest(self) -> <Self as SegmentCollector>::Fruit {- self.stats.non_zero_count()- }-}--fn main() -> tantivy::Result<()> {- // # Defining the schema- //- // The Tantivy index requires a very strict schema.- // The schema declares which fields are in the index,- // and for each field, its type and "the way it should- // be indexed".-- // first we need to define a schema ...- let mut schema_builder = Schema::builder();-- // We'll assume a fictional index containing- // products, and with a name, a description, and a price.- let product_name = schema_builder.add_text_field("name", TEXT);- let product_description = schema_builder.add_text_field("description", TEXT);- let price = schema_builder.add_u64_field("price", INDEXED | FAST);- let schema = schema_builder.build();-- // # Indexing documents- //- // Lets index a bunch of fake documents for the sake of- // this example.- let index = Index::create_in_ram(schema);-- let mut index_writer: IndexWriter = index.writer(50_000_000)?;- index_writer.add_document(doc!(- product_name => "Super Broom 2000",- product_description => "While it is ok for short distance travel, this broom \- was designed quiditch. It will up your game.",- price => 30_200u64- ))?;- index_writer.add_document(doc!(- product_name => "Turbulobroom",- product_description => "You might have heard of this broom before : it is the sponsor of the Wales team.\- You'll enjoy its sharp turns, and rapid acceleration",- price => 29_240u64- ))?;- index_writer.add_document(doc!(- product_name => "Broomio",- product_description => "Great value for the price. This broom is a market favorite",- price => 21_240u64- ))?;- index_writer.add_document(doc!(- product_name => "Whack a Mole",- product_description => "Prime quality bat.",- price => 5_200u64- ))?;- index_writer.commit()?;-- let reader = index.reader()?;- let searcher = reader.searcher();- let query_parser = QueryParser::for_index(&index, vec![product_name, product_description]);-- // here we want to search for `broom` and use `StatsCollector` on the hits.- let query = query_parser.parse_query("broom")?;- if let Some(stats) =- searcher.search(&query, &StatsCollector::with_field("price".to_string()))?- {- println!("count: {}", stats.count());- println!("mean: {}", stats.mean());- println!("standard deviation: {}", stats.standard_deviation());- }-- Ok(())-}diff --git a/crates/tantivy/examples/custom_tokenizer.rs b/crates/tantivy/examples/custom_tokenizer.rsdeleted file mode 100644index 6ec6047c..00000000--- a/crates/tantivy/examples/custom_tokenizer.rs+++ /dev/null@@ -1,111 +0,0 @@-// # Defining a tokenizer pipeline-//-// In this example, we'll see how to define a tokenizer-// by creating a custom `NgramTokenizer`.-use tantivy::collector::TopDocs;-use tantivy::query::QueryParser;-use tantivy::schema::*;-use tantivy::tokenizer::NgramTokenizer;-use tantivy::{doc, Index, IndexWriter};--fn main() -> tantivy::Result<()> {- // # Defining the schema- //- // The Tantivy index requires a very strict schema.- // The schema declares which fields are in the index,- // and for each field, its type and "the way it should- // be indexed".-- // first we need to define a schema ...- let mut schema_builder = Schema::builder();-- // Our first field is title.- // In this example we want to use NGram searching- // we will set that to 3 characters, so any three- // char in the title should be findable.- let text_field_indexing = TextFieldIndexing::default()- .set_tokenizer("ngram3")- .set_index_option(IndexRecordOption::WithFreqsAndPositions);- let text_options = TextOptions::default()- .set_indexing_options(text_field_indexing)- .set_stored();- let title = schema_builder.add_text_field("title", text_options);-- // Our second field is body.- // We want full-text search for it, but we do not- // need to be able to be able to retrieve it- // for our application.- //- // We can make our index lighter by omitting the `STORED` flag.- let body = schema_builder.add_text_field("body", TEXT);-- let schema = schema_builder.build();-- // # Indexing documents- //- // Let's create a brand new index.- // To simplify we will work entirely in RAM.- // This is not what you want in reality, but it is very useful- // for your unit tests... Or this example.- let index = Index::create_in_ram(schema.clone());-- // here we are registering our custom tokenizer- // this will store tokens of 3 characters each- index- .tokenizers()- .register("ngram3", NgramTokenizer::new(3, 3, false).unwrap());-- // To insert document we need an index writer.- // There must be only one writer at a time.- // This single `IndexWriter` is already- // multithreaded.- //- // Here we use a buffer of 50MB per thread. Using a bigger- // memory arena for the indexer can increase its throughput.- let mut index_writer: IndexWriter = index.writer(50_000_000)?;- index_writer.add_document(doc!(- title => "The Old Man and the Sea",- body => "He was an old man who fished alone in a skiff in the Gulf Stream and \- he had gone eighty-four days now without taking a fish."- ))?;- index_writer.add_document(doc!(- title => "Of Mice and Men",- body => r#"A few miles south of Soledad, the Salinas River drops in close to the hillside- bank and runs deep and green. The water is warm too, for it has slipped twinkling- over the yellow sands in the sunlight before reaching the narrow pool. On one- side of the river the golden foothill slopes curve up to the strong and rocky- Gabilan Mountains, but on the valley side the water is lined with trees—willows- fresh and green with every spring, carrying in their lower leaf junctures the- debris of the winter’s flooding; and sycamores with mottled, white, recumbent- limbs and branches that arch over the pool"#- ))?;- index_writer.add_document(doc!(- title => "Frankenstein",- body => r#"You will rejoice to hear that no disaster has accompanied the commencement of an- enterprise which you have regarded with such evil forebodings. I arrived here- yesterday, and my first task is to assure my dear sister of my welfare and- increasing confidence in the success of my undertaking."#- ))?;- index_writer.commit()?;-- let reader = index.reader()?;- let searcher = reader.searcher();-- // The query parser can interpret human queries.- // Here, if the user does not specify which- // field they want to search, tantivy will search- // in both title and body.- let query_parser = QueryParser::for_index(&index, vec![title, body]);-- // here we want to get a hit on the 'ken' in Frankenstein- let query = query_parser.parse_query("ken")?;-- let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;-- for (_, doc_address) in top_docs {- let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;- println!("{}", retrieved_doc.to_json(&schema));- }-- Ok(())-}diff --git a/crates/tantivy/examples/date_time_field.rs b/crates/tantivy/examples/date_time_field.rsdeleted file mode 100644index c00a22f0..00000000--- a/crates/tantivy/examples/date_time_field.rs+++ /dev/null@@ -1,77 +0,0 @@-// # DateTime field example-//-// This example shows how the DateTime field can be used--use tantivy::collector::TopDocs;-use tantivy::query::QueryParser;-use tantivy::schema::{DateOptions, Document, Schema, Value, INDEXED, STORED, STRING};-use tantivy::{Index, IndexWriter, TantivyDocument};--fn main() -> tantivy::Result<()> {- // # Defining the schema- let mut schema_builder = Schema::builder();- let opts = DateOptions::from(INDEXED)- .set_stored()- .set_fast()- .set_precision(tantivy::schema::DateTimePrecision::Seconds);- // Add `occurred_at` date field type- let occurred_at = schema_builder.add_date_field("occurred_at", opts);- let event_type = schema_builder.add_text_field("event", STRING | STORED);- let schema = schema_builder.build();-- // # Indexing documents- let index = Index::create_in_ram(schema.clone());-- let mut index_writer: IndexWriter = index.writer(50_000_000)?;- // The dates are passed as string in the RFC3339 format- let doc = TantivyDocument::parse_json(- &schema,- r#"{- "occurred_at": "2022-06-22T12:53:50.53Z",- "event": "pull-request"- }"#,- )?;- index_writer.add_document(doc)?;- let doc = TantivyDocument::parse_json(- &schema,- r#"{- "occurred_at": "2022-06-22T13:00:00.22Z",- "event": "comment"- }"#,- )?;- index_writer.add_document(doc)?;- index_writer.commit()?;-- let reader = index.reader()?;- let searcher = reader.searcher();-- // # Search- let query_parser = QueryParser::for_index(&index, vec![event_type]);- {- // Simple exact search on the date- let query = query_parser.parse_query("occurred_at:\"2022-06-22T12:53:50.53Z\"")?;- let count_docs = searcher.search(&*query, &TopDocs::with_limit(5))?;- assert_eq!(count_docs.len(), 1);- }- {- // Range query on the date field- let query = query_parser- .parse_query(r#"occurred_at:[2022-06-22T12:58:00Z TO 2022-06-23T00:00:00Z}"#)?;- let count_docs = searcher.search(&*query, &TopDocs::with_limit(4))?;- assert_eq!(count_docs.len(), 1);- for (_score, doc_address) in count_docs {- let retrieved_doc = searcher.doc::<TantivyDocument>(doc_address)?;- assert!(retrieved_doc- .get_first(occurred_at)- .unwrap()- .as_value()- .as_datetime()- .is_some(),);- assert_eq!(- retrieved_doc.to_json(&schema),- r#"{"event":["comment"],"occurred_at":["2022-06-22T13:00:00.22Z"]}"#- );- }- }- Ok(())-}diff --git a/crates/tantivy/examples/deleting_updating_documents.rs b/crates/tantivy/examples/deleting_updating_documents.rsdeleted file mode 100644index ac9859a4..00000000--- a/crates/tantivy/examples/deleting_updating_documents.rs+++ /dev/null@@ -1,144 +0,0 @@-// # Deleting and Updating (?) documents-//-// This example explains how to delete and update documents.-// In fact there is actually no such thing as an update in tantivy.-//-// To update a document, you need to delete a document and then reinsert-// its new version.-//-// ----// Importing tantivy...-use tantivy::collector::TopDocs;-use tantivy::query::TermQuery;-use tantivy::schema::*;-use tantivy::{doc, Index, IndexReader, IndexWriter};--// A simple helper function to fetch a single document-// given its id from our index.-// It will be helpful to check our work.-fn extract_doc_given_isbn(- reader: &IndexReader,- isbn_term: &Term,-) -> tantivy::Result<Option<TantivyDocument>> {- let searcher = reader.searcher();-- // This is the simplest query you can think of.- // It matches all of the documents containing a specific term.- //- // The second argument is here to tell we don't care about decoding positions,- // or term frequencies.- let term_query = TermQuery::new(isbn_term.clone(), IndexRecordOption::Basic);- let top_docs = searcher.search(&term_query, &TopDocs::with_limit(1))?;-- if let Some((_score, doc_address)) = top_docs.first() {- let doc = searcher.doc(*doc_address)?;- Ok(Some(doc))- } else {- // no doc matching this ID.- Ok(None)- }-}--fn main() -> tantivy::Result<()> {- // # Defining the schema- //- // Check out the *basic_search* example if this makes- // small sense to you.- let mut schema_builder = Schema::builder();-- // Tantivy does not really have a notion of primary id.- // This may change in the future.- //- // Still, we can create a `isbn` field and use it as an id. This- // field can be `u64` or a `text`, depending on your use case.- // It just needs to be indexed.- //- // If it is `text`, let's make sure to keep it `raw` and let's avoid- // running any text processing on it.- // This is done by associating this field to the tokenizer named `raw`.- // Rather than building our- // [`TextOptions`](//docs.rs/tantivy/~0/tantivy/schema/struct.TextOptions.html) manually, We- // use the `STRING` shortcut. `STRING` stands for indexed (without term frequency or positions)- // and untokenized.- //- // Because we also want to be able to see this `id` in our returned documents,- // we also mark the field as stored.- let isbn = schema_builder.add_text_field("isbn", STRING | STORED);- let title = schema_builder.add_text_field("title", TEXT | STORED);- let schema = schema_builder.build();-- let index = Index::create_in_ram(schema.clone());-- let mut index_writer: IndexWriter = index.writer(50_000_000)?;-- // Let's add a couple of documents, for the sake of the example.- let mut old_man_doc = TantivyDocument::default();- old_man_doc.add_text(title, "The Old Man and the Sea");- index_writer.add_document(doc!(- isbn => "978-0099908401",- title => "The old Man and the see"- ))?;- index_writer.add_document(doc!(- isbn => "978-0140177398",- title => "Of Mice and Men",- ))?;- index_writer.add_document(doc!(- title => "Frankentein", //< Oops there is a typo here.- isbn => "978-9176370711",- ))?;- index_writer.commit()?;- let reader = index.reader()?;-- let frankenstein_isbn = Term::from_field_text(isbn, "978-9176370711");-- // Oops our frankenstein doc seems misspelled- let frankenstein_doc_misspelled = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();- assert_eq!(- frankenstein_doc_misspelled.to_json(&schema),- r#"{"isbn":["978-9176370711"],"title":["Frankentein"]}"#,- );-- // # Update = Delete + Insert- //- // Here we will want to update the typo in the `Frankenstein` book.- //- // Tantivy does not handle updates directly, we need to delete- // and reinsert the document.- //- // This can be complicated as it means you need to have access- // to the entire document. It is good practise to integrate tantivy- // with a key value store for this reason.- //- // To remove one of the document, we just call `delete_term`- // on its id.- //- // Note that `tantivy` does nothing to enforce the idea that- // there is only one document associated with this id.- //- // Also you might have noticed that we apply the delete before- // having committed. This does not matter really...- index_writer.delete_term(frankenstein_isbn.clone());-- // We now need to reinsert our document without the typo.- index_writer.add_document(doc!(- title => "Frankenstein",- isbn => "978-9176370711",- ))?;-- // You are guaranteed that your clients will only observe your index in- // the state it was in after a commit.- // In this example, your search engine will at no point be missing the *Frankenstein* document.- // Everything happened as if the document was updated.- index_writer.commit()?;- // We reload our searcher to make our change available to clients.- reader.reload()?;-- // No more typo!- let frankenstein_new_doc = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();- assert_eq!(- frankenstein_new_doc.to_json(&schema),- r#"{"isbn":["978-9176370711"],"title":["Frankenstein"]}"#,- );-- Ok(())-}diff --git a/crates/tantivy/examples/faceted_search.rs b/crates/tantivy/examples/faceted_search.rsdeleted file mode 100644index 216f55f7..00000000--- a/crates/tantivy/examples/faceted_search.rs+++ /dev/null@@ -1,114 +0,0 @@-// # Faceted Search-//-// This example covers the faceted search functionalities of-// tantivy.-//-// We will :-// - define a text field "name" in our schema-// - define a facet field "classification" in our schema-// - create an index in memory-// - index few documents with respective facets in our index-// - search and count the number of documents that the classifications start the facet "/Felidae"-// - Search the facet "/Felidae/Pantherinae" and count the number of documents that the-// classifications include the facet.-//-// ----// Importing tantivy...-use tantivy::collector::FacetCollector;-use tantivy::query::{AllQuery, TermQuery};-use tantivy::schema::*;-use tantivy::{doc, Index, IndexWriter};--fn main() -> tantivy::Result<()> {- // Let's create a temporary directory for the sake of this example- let mut schema_builder = Schema::builder();-- let name = schema_builder.add_text_field("name", TEXT | STORED);- // this is our faceted field: its scientific classification- let classification = schema_builder.add_facet_field("classification", FacetOptions::default());-- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);-- let mut index_writer: IndexWriter = index.writer(30_000_000)?;-- // For convenience, tantivy also comes with a macro to- // reduce the boilerplate above.- index_writer.add_document(doc!(- name => "Cat",- classification => Facet::from("/Felidae/Felinae/Felis")- ))?;- index_writer.add_document(doc!(- name => "Canada lynx",- classification => Facet::from("/Felidae/Felinae/Lynx")- ))?;- index_writer.add_document(doc!(- name => "Cheetah",- classification => Facet::from("/Felidae/Felinae/Acinonyx")- ))?;- index_writer.add_document(doc!(- name => "Tiger",- classification => Facet::from("/Felidae/Pantherinae/Panthera")- ))?;- index_writer.add_document(doc!(- name => "Lion",- classification => Facet::from("/Felidae/Pantherinae/Panthera")- ))?;- index_writer.add_document(doc!(- name => "Jaguar",- classification => Facet::from("/Felidae/Pantherinae/Panthera")- ))?;- index_writer.add_document(doc!(- name => "Sunda clouded leopard",- classification => Facet::from("/Felidae/Pantherinae/Neofelis")- ))?;- index_writer.add_document(doc!(- name => "Fossa",- classification => Facet::from("/Eupleridae/Cryptoprocta")- ))?;- index_writer.commit()?;-- let reader = index.reader()?;- let searcher = reader.searcher();- {- let mut facet_collector = FacetCollector::for_field("classification");- facet_collector.add_facet("/Felidae");- let facet_counts = searcher.search(&AllQuery, &facet_collector)?;- // This lists all of the facet counts, right below "/Felidae".- let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae").collect();- assert_eq!(- facets,- vec![- (&Facet::from("/Felidae/Felinae"), 3),- (&Facet::from("/Felidae/Pantherinae"), 4),- ]- );- }-- // Facets are also searchable.- //- // For instance a common UI pattern is to allow the user someone to click on a facet link- // (e.g: `Pantherinae`) to drill down and filter the current result set with this subfacet.- //- // The search would then look as follows.-- // Check the reference doc for different ways to create a `Facet` object.- {- let facet = Facet::from("/Felidae/Pantherinae");- let facet_term = Term::from_facet(classification, &facet);- let facet_term_query = TermQuery::new(facet_term, IndexRecordOption::Basic);- let mut facet_collector = FacetCollector::for_field("classification");- facet_collector.add_facet("/Felidae/Pantherinae");- let facet_counts = searcher.search(&facet_term_query, &facet_collector)?;- let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae/Pantherinae").collect();- assert_eq!(- facets,- vec![- (&Facet::from("/Felidae/Pantherinae/Neofelis"), 1),- (&Facet::from("/Felidae/Pantherinae/Panthera"), 3),- ]- );- }-- Ok(())-}diff --git a/crates/tantivy/examples/faceted_search_with_tweaked_score.rs b/crates/tantivy/examples/faceted_search_with_tweaked_score.rsdeleted file mode 100644index d21a1c3d..00000000--- a/crates/tantivy/examples/faceted_search_with_tweaked_score.rs+++ /dev/null@@ -1,104 +0,0 @@-// # Faceted Search With Tweak Score-//-// This example covers the faceted search functionalities of-// tantivy.-//-// We will :-// - define a text field "name" in our schema-// - define a facet field "classification" in our schema--use std::collections::HashSet;--use tantivy::collector::TopDocs;-use tantivy::query::BooleanQuery;-use tantivy::schema::*;-use tantivy::{doc, DocId, Index, IndexWriter, Score, SegmentReader};--fn main() -> tantivy::Result<()> {- let mut schema_builder = Schema::builder();-- let title = schema_builder.add_text_field("title", STORED);- let ingredient = schema_builder.add_facet_field("ingredient", FacetOptions::default());-- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);-- let mut index_writer: IndexWriter = index.writer(30_000_000)?;-- index_writer.add_document(doc!(- title => "Fried egg",- ingredient => Facet::from("/ingredient/egg"),- ingredient => Facet::from("/ingredient/oil"),- ))?;- index_writer.add_document(doc!(- title => "Scrambled egg",- ingredient => Facet::from("/ingredient/egg"),- ingredient => Facet::from("/ingredient/butter"),- ingredient => Facet::from("/ingredient/milk"),- ingredient => Facet::from("/ingredient/salt"),- ))?;- index_writer.add_document(doc!(- title => "Egg rolls",- ingredient => Facet::from("/ingredient/egg"),- ingredient => Facet::from("/ingredient/garlic"),- ingredient => Facet::from("/ingredient/salt"),- ingredient => Facet::from("/ingredient/oil"),- ingredient => Facet::from("/ingredient/tortilla-wrap"),- ingredient => Facet::from("/ingredient/mushroom"),- ))?;- index_writer.commit()?;-- let reader = index.reader()?;- let searcher = reader.searcher();- {- let facets = [- Facet::from("/ingredient/egg"),- Facet::from("/ingredient/oil"),- Facet::from("/ingredient/garlic"),- Facet::from("/ingredient/mushroom"),- ];- let query = BooleanQuery::new_multiterms_query(- facets- .iter()- .map(|key| Term::from_facet(ingredient, key))- .collect(),- );- let top_docs_by_custom_score =- // Call TopDocs with a custom tweak score- TopDocs::with_limit(2).tweak_score(move |segment_reader: &SegmentReader| {- let ingredient_reader = segment_reader.facet_reader("ingredient").unwrap();- let facet_dict = ingredient_reader.facet_dict();-- let query_ords: HashSet<u64> = facets- .iter()- .filter_map(|key| facet_dict.term_ord(key.encoded_str()).unwrap())- .collect();-- move |doc: DocId, original_score: Score| {- // Update the original score with a tweaked score- let missing_ingredients = ingredient_reader- .facet_ords(doc)- .filter(|ord| !query_ords.contains(ord))- .count();- let tweak = 1.0 / 4_f32.powi(missing_ingredients as i32);-- original_score * tweak- }- });- let top_docs = searcher.search(&query, &top_docs_by_custom_score)?;-- let titles: Vec<String> = top_docs- .iter()- .map(|(_, doc_id)| {- searcher- .doc::<TantivyDocument>(*doc_id)- .unwrap()- .get_first(title)- .and_then(|v| v.as_str().map(|el| el.to_string()))- .unwrap()- })- .collect();- assert_eq!(titles, vec!["Fried egg", "Egg rolls"]);- }- Ok(())-}diff --git a/crates/tantivy/examples/fuzzy_search.rs b/crates/tantivy/examples/fuzzy_search.rsdeleted file mode 100644index ab681a7a..00000000--- a/crates/tantivy/examples/fuzzy_search.rs+++ /dev/null@@ -1,167 +0,0 @@-// # Basic Example-//-// This example covers the basic functionalities of-// tantivy.-//-// We will :-// - define our schema-// - create an index in a directory-// - index a few documents into our index-// - search for the best document matching a basic query-// - retrieve the best document's original content.-// ----// Importing tantivy...-use tantivy::collector::{Count, TopDocs};-use tantivy::query::FuzzyTermQuery;-use tantivy::schema::*;-use tantivy::{doc, Index, IndexWriter, ReloadPolicy};-use tempfile::TempDir;--fn main() -> tantivy::Result<()> {- // Let's create a temporary directory for the- // sake of this example- let index_path = TempDir::new()?;-- // # Defining the schema- //- // The Tantivy index requires a very strict schema.- // The schema declares which fields are in the index,- // and for each field, its type and "the way it should- // be indexed".-- // First we need to define a schema ...- let mut schema_builder = Schema::builder();-- // Our first field is title.- // We want full-text search for it, and we also want- // to be able to retrieve the document after the search.- //- // `TEXT | STORED` is some syntactic sugar to describe- // that.- //- // `TEXT` means the field should be tokenized and indexed,- // along with its term frequency and term positions.- //- // `STORED` means that the field will also be saved- // in a compressed, row-oriented key-value store.- // This store is useful for reconstructing the- // documents that were selected during the search phase.- let title = schema_builder.add_text_field("title", TEXT | STORED);-- let schema = schema_builder.build();-- // # Indexing documents- //- // Let's create a brand new index.- //- // This will actually just save a meta.json- // with our schema in the directory.- let index = Index::create_in_dir(&index_path, schema.clone())?;-- // To insert a document we will need an index writer.- // There must be only one writer at a time.- // This single `IndexWriter` is already- // multithreaded.- //- // Here we give tantivy a budget of `50MB`.- // Using a bigger memory_arena for the indexer may increase- // throughput, but 50 MB is already plenty.- let mut index_writer: IndexWriter = index.writer(50_000_000)?;-- // Let's index our documents!- // We first need a handle on the title and the body field.-- // ### Adding documents- //- index_writer.add_document(doc!(- title => "The Name of the Wind",- ))?;- index_writer.add_document(doc!(- title => "The Diary of Muadib",- ))?;- index_writer.add_document(doc!(- title => "A Dairy Cow",- ))?;- index_writer.add_document(doc!(- title => "The Diary of a Young Girl",- ))?;- index_writer.commit()?;-- // ### Committing- //- // At this point our documents are not searchable.- //- //- // We need to call `.commit()` explicitly to force the- // `index_writer` to finish processing the documents in the queue,- // flush the current index to the disk, and advertise- // the existence of new documents.- //- // This call is blocking.- index_writer.commit()?;-- // If `.commit()` returns correctly, then all of the- // documents that have been added are guaranteed to be- // persistently indexed.- //- // In the scenario of a crash or a power failure,- // tantivy behaves as if it has rolled back to its last- // commit.-- // # Searching- //- // ### Searcher- //- // A reader is required first in order to search an index.- // It acts as a `Searcher` pool that reloads itself,- // depending on a `ReloadPolicy`.- //- // For a search server you will typically create one reader for the entire lifetime of your- // program, and acquire a new searcher for every single request.- //- // In the code below, we rely on the 'ON_COMMIT' policy: the reader- // will reload the index automatically after each commit.- let reader = index- .reader_builder()- .reload_policy(ReloadPolicy::OnCommitWithDelay)- .try_into()?;-- // We now need to acquire a searcher.- //- // A searcher points to a snapshotted, immutable version of the index.- //- // Some search experience might require more than- // one query. Using the same searcher ensures that all of these queries will run on the- // same version of the index.- //- // Acquiring a `searcher` is very cheap.- //- // You should acquire a searcher every time you start processing a request and- // and release it right after your query is finished.- let searcher = reader.searcher();-- // ### FuzzyTermQuery- {- let term = Term::from_field_text(title, "Diary");- let query = FuzzyTermQuery::new(term, 2, true);-- let (top_docs, count) = searcher- .search(&query, &(TopDocs::with_limit(5), Count))- .unwrap();- assert_eq!(count, 3);- assert_eq!(top_docs.len(), 3);- for (score, doc_address) in top_docs {- // Note that the score is not lower for the fuzzy hit.- // There's an issue open for that: https://github.com/quickwit-oss/tantivy/issues/563- let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;- println!("score {score:?} doc {}", retrieved_doc.to_json(&schema));- // score 1.0 doc {"title":["The Diary of Muadib"]}- //- // score 1.0 doc {"title":["The Diary of a Young Girl"]}- //- // score 1.0 doc {"title":["A Dairy Cow"]}- }- }-- Ok(())-}diff --git a/crates/tantivy/examples/index_from_multiple_threads.rs b/crates/tantivy/examples/index_from_multiple_threads.rsdeleted file mode 100644index 288d306d..00000000--- a/crates/tantivy/examples/index_from_multiple_threads.rs+++ /dev/null@@ -1,104 +0,0 @@-// # Indexing from different threads.-//-// It is fairly common to have to index from different threads.-// Tantivy forbids to create more than one `IndexWriter` at a time.-//-// This `IndexWriter` itself has its own multithreaded layer, so managing your own-// indexing threads will not help. However, it can still be useful for some applications.-//-// For instance, if preparing documents to send to tantivy before indexing is the bottleneck of-// your application, it is reasonable to have multiple threads.-//-// Another very common reason to want to index from multiple threads, is implementing a webserver-// with CRUD capabilities. The server framework will most likely handle request from-// different threads.-//-// The recommended way to address both of these use case is to wrap your `IndexWriter` into a-// `Arc<RwLock<IndexWriter>>`.-//-// While this is counterintuitive, adding and deleting documents do not require mutability-// over the `IndexWriter`, so several threads will be able to do this operation concurrently.-//-// The example below does not represent an actual real-life use case (who would spawn thread to-// index a single document?), but aims at demonstrating the mechanism that makes indexing-// from several threads possible.--// ----// Importing tantivy...-use std::sync::{Arc, RwLock};-use std::thread;-use std::time::Duration;--use tantivy::schema::{Schema, STORED, TEXT};-use tantivy::{doc, Index, IndexWriter, Opstamp, TantivyError};--fn main() -> tantivy::Result<()> {- // # Defining the schema- let mut schema_builder = Schema::builder();- let title = schema_builder.add_text_field("title", TEXT | STORED);- let body = schema_builder.add_text_field("body", TEXT);- let schema = schema_builder.build();-- let index = Index::create_in_ram(schema);- let index_writer: Arc<RwLock<IndexWriter>> = Arc::new(RwLock::new(index.writer(50_000_000)?));-- // # First indexing thread.- let index_writer_clone_1 = index_writer.clone();- thread::spawn(move || {- // we index 100 times the document... for the sake of the example.- for i in 0..100 {- let opstamp = index_writer_clone_1- .read().unwrap() //< A read lock is sufficient here.- .add_document(- doc!(- title => "Of Mice and Men",- body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \- bank and runs deep and green. The water is warm too, for it has slipped twinkling \- over the yellow sands in the sunlight before reaching the narrow pool. On one \- side of the river the golden foothill slopes curve up to the strong and rocky \- Gabilan Mountains, but on the valley side the water is lined with trees—willows \- fresh and green with every spring, carrying in their lower leaf junctures the \- debris of the winter’s flooding; and sycamores with mottled, white, recumbent \- limbs and branches that arch over the pool"- ))?;- println!("add doc {i} from thread 1 - opstamp {opstamp}");- thread::sleep(Duration::from_millis(20));- }- Result::<(), TantivyError>::Ok(())- });-- // # Second indexing thread.- let index_writer_clone_2 = index_writer.clone();- // For convenience, tantivy also comes with a macro to- // reduce the boilerplate above.- thread::spawn(move || {- // we index 100 times the document... for the sake of the example.- for i in 0..100 {- // A read lock is sufficient here.- let opstamp = {- let index_writer_rlock = index_writer_clone_2.read().unwrap();- index_writer_rlock.add_document(doc!(- title => "Manufacturing consent",- body => "Some great book description..."- ))?- };- println!("add doc {i} from thread 2 - opstamp {opstamp}");- thread::sleep(Duration::from_millis(10));- }- Result::<(), TantivyError>::Ok(())- });-- // # In the main thread, we commit 10 times, once every 500ms.- for _ in 0..10 {- let opstamp: Opstamp = {- // Committing or rollbacking on the other hand requires write lock. This will block- // other threads.- let mut index_writer_wlock = index_writer.write().unwrap();- index_writer_wlock.commit()?- };- println!("committed with opstamp {opstamp}");- thread::sleep(Duration::from_millis(500));- }-- Ok(())-}diff --git a/crates/tantivy/examples/index_with_json.rs b/crates/tantivy/examples/index_with_json.rsdeleted file mode 100644index 5869234f..00000000--- a/crates/tantivy/examples/index_with_json.rs+++ /dev/null@@ -1,40 +0,0 @@-use tantivy::schema::*;--// # Document from json-//-// For convenience, `Document` can be parsed directly from json.-fn main() -> tantivy::Result<()> {- // Let's first define a schema and an index.- // Check out the basic example if this is confusing to you.- //- // first we need to define a schema ...- let mut schema_builder = Schema::builder();- schema_builder.add_text_field("title", TEXT | STORED);- schema_builder.add_text_field("body", TEXT);- schema_builder.add_u64_field("year", INDEXED);- let schema = schema_builder.build();-- // Let's assume we have a json-serialized document.- let mice_and_men_doc_json = r#"{- "title": "Of Mice and Men",- "year": 1937- }"#;-- // We can parse our document- let _mice_and_men_doc = TantivyDocument::parse_json(&schema, mice_and_men_doc_json)?;-- // Multi-valued field are allowed, they are- // expressed in JSON by an array.- // The following document has two titles.- let frankenstein_json = r#"{- "title": ["Frankenstein", "The Modern Prometheus"],- "year": 1818- }"#;- let _frankenstein_doc = TantivyDocument::parse_json(&schema, frankenstein_json)?;-- // Note that the schema is saved in your index directory.- //- // As a result, Indexes are aware of their schema, and you can use this feature- // just by opening an existing `Index`, and calling `index.schema()..parse_document(json)`.- Ok(())-}diff --git a/crates/tantivy/examples/integer_range_search.rs b/crates/tantivy/examples/integer_range_search.rsdeleted file mode 100644index 10f2015c..00000000--- a/crates/tantivy/examples/integer_range_search.rs+++ /dev/null@@ -1,35 +0,0 @@-// # Searching a range on an indexed int field.-//-// Below is an example of creating an indexed integer field in your schema-// You can use RangeQuery to get a Count of all occurrences in a given range.-use tantivy::collector::Count;-use tantivy::query::RangeQuery;-use tantivy::schema::{Schema, INDEXED};-use tantivy::{doc, Index, IndexWriter, Result};--fn main() -> Result<()> {- // For the sake of simplicity, this schema will only have 1 field- let mut schema_builder = Schema::builder();-- // `INDEXED` is a short-hand to indicate that our field should be "searchable".- let year_field = schema_builder.add_u64_field("year", INDEXED);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let reader = index.reader()?;- {- let mut index_writer: IndexWriter = index.writer_with_num_threads(1, 6_000_000)?;- for year in 1950u64..2019u64 {- index_writer.add_document(doc!(year_field => year))?;- }- index_writer.commit()?;- // The index will be a range of years- }- reader.reload()?;- let searcher = reader.searcher();- // The end is excluded i.e. here we are searching up to 1969- let docs_in_the_sixties = RangeQuery::new_u64("year".to_string(), 1960..1970);- // Uses a Count collector to sum the total number of docs in the range- let num_60s_books = searcher.search(&docs_in_the_sixties, &Count)?;- assert_eq!(num_60s_books, 10);- Ok(())-}diff --git a/crates/tantivy/examples/ip_field.rs b/crates/tantivy/examples/ip_field.rsdeleted file mode 100644index 17caf55e..00000000--- a/crates/tantivy/examples/ip_field.rs+++ /dev/null@@ -1,110 +0,0 @@-// # IP Address example-//-// This example shows how the ip field can be used-// with IpV6 and IpV4.--use tantivy::collector::{Count, TopDocs};-use tantivy::query::QueryParser;-use tantivy::schema::{Schema, FAST, INDEXED, STORED, STRING};-use tantivy::{Index, IndexWriter, TantivyDocument};--fn main() -> tantivy::Result<()> {- // # Defining the schema- // We set the IP field as `INDEXED`, so it can be searched- // `FAST` will create a fast field. The fast field will be used to execute search queries.- // `FAST` is not a requirement for range queries, it can also be executed on the inverted index- // which is created by `INDEXED`.- let mut schema_builder = Schema::builder();- let event_type = schema_builder.add_text_field("event_type", STRING | STORED);- let ip = schema_builder.add_ip_addr_field("ip", STORED | INDEXED | FAST);- let schema = schema_builder.build();-- // # Indexing documents- let index = Index::create_in_ram(schema.clone());-- let mut index_writer: IndexWriter = index.writer(50_000_000)?;-- // ### IPv4- // Adding documents that contain an IPv4 address. Notice that the IP addresses are passed as- // `String`. Since the field is of type ip, we parse the IP address from the string and store it- // internally as IPv6.- let doc = TantivyDocument::parse_json(- &schema,- r#"{- "ip": "192.168.0.33",- "event_type": "login"- }"#,- )?;- index_writer.add_document(doc)?;- let doc = TantivyDocument::parse_json(- &schema,- r#"{- "ip": "192.168.0.80",- "event_type": "checkout"- }"#,- )?;- index_writer.add_document(doc)?;- // ### IPv6- // Adding a document that contains an IPv6 address.- let doc = TantivyDocument::parse_json(- &schema,- r#"{- "ip": "2001:0db8:85a3:0000:0000:8a2e:0370:7334",- "event_type": "checkout"- }"#,- )?;-- index_writer.add_document(doc)?;- // Commit will create a segment containing our documents.- index_writer.commit()?;-- let reader = index.reader()?;- let searcher = reader.searcher();-- // # Search- // Range queries on IPv4. Since we created a fast field, the fast field will be used to execute- // the search.- // ### Range Queries- let query_parser = QueryParser::for_index(&index, vec![event_type, ip]);- {- // Inclusive range queries- let query = query_parser.parse_query("ip:[192.168.0.80 TO 192.168.0.100]")?;- let count_docs = searcher.search(&*query, &TopDocs::with_limit(5))?;- assert_eq!(count_docs.len(), 1);- }- {- // Exclusive range queries- let query = query_parser.parse_query("ip:{192.168.0.80 TO 192.168.1.100]")?;- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2))?;- assert_eq!(count_docs.len(), 0);- }- {- // Find docs with IP addresses smaller equal 192.168.1.100- let query = query_parser.parse_query("ip:[* TO 192.168.1.100]")?;- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2))?;- assert_eq!(count_docs.len(), 2);- }- {- // Find docs with IP addresses smaller than 192.168.1.100- let query = query_parser.parse_query("ip:[* TO 192.168.1.100}")?;- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2))?;- assert_eq!(count_docs.len(), 2);- }-- // ### Exact Queries- // Exact search on IPv4.- {- let query = query_parser.parse_query("ip:192.168.0.80")?;- let count_docs = searcher.search(&*query, &Count)?;- assert_eq!(count_docs, 1);- }- // Exact search on IPv6.- // IpV6 addresses need to be quoted because they contain `:`- {- let query = query_parser.parse_query("ip:\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\"")?;- let count_docs = searcher.search(&*query, &Count)?;- assert_eq!(count_docs, 1);- }-- Ok(())-}diff --git a/crates/tantivy/examples/iterating_docs_and_positions.rs b/crates/tantivy/examples/iterating_docs_and_positions.rsdeleted file mode 100644index b5827661..00000000--- a/crates/tantivy/examples/iterating_docs_and_positions.rs+++ /dev/null@@ -1,136 +0,0 @@-// # Iterating docs and positions.-//-// At its core of tantivy, relies on a data structure-// called an inverted index.-//-// This example shows how to manually iterate through-// the list of documents containing a term, getting-// its term frequency, and accessing its positions.--use tantivy::postings::Postings;-// ----// Importing tantivy...-use tantivy::schema::*;-use tantivy::{doc, DocSet, Index, IndexWriter, TERMINATED};--fn main() -> tantivy::Result<()> {- // We first create a schema for the sake of the- // example. Check the `basic_search` example for more information.- let mut schema_builder = Schema::builder();-- // For this example, we need to make sure to index positions for our title- // field. `TEXT` precisely does this.- let title = schema_builder.add_text_field("title", TEXT | STORED);- let schema = schema_builder.build();-- let index = Index::create_in_ram(schema);-- let mut index_writer: IndexWriter = index.writer_with_num_threads(1, 50_000_000)?;- index_writer.add_document(doc!(title => "The Old Man and the Sea"))?;- index_writer.add_document(doc!(title => "Of Mice and Men"))?;- index_writer.add_document(doc!(title => "The modern Promotheus"))?;- index_writer.commit()?;-- let reader = index.reader()?;-- let searcher = reader.searcher();-- // A tantivy index is actually a collection of segments.- // Similarly, a searcher just wraps a list `segment_reader`.- //- // (Because we indexed a very small number of documents over one thread- // there is actually only one segment here, but let's iterate through the list- // anyway)- for segment_reader in searcher.segment_readers() {- // A segment contains different data structure.- // Inverted index stands for the combination of- // - the term dictionary- // - the inverted lists associated with each terms and their positions- let inverted_index = segment_reader.inverted_index(title)?;-- // A `Term` is a text token associated with a field.- // Let's go through all docs containing the term `title:the` and access their position- let term_the = Term::from_field_text(title, "the");-- // This segment posting object is like a cursor over the documents matching the term.- // The `IndexRecordOption` arguments tells tantivy we will be interested in both term- // frequencies and positions.- //- // If you don't need all this information, you may get better performance by decompressing- // less information.- if let Some(mut segment_postings) =- inverted_index.read_postings(&term_the, IndexRecordOption::WithFreqsAndPositions)?- {- // this buffer will be used to request for positions- let mut positions: Vec<u32> = Vec::with_capacity(100);- let mut doc_id = segment_postings.doc();- while doc_id != TERMINATED {- // This MAY contains deleted documents as well.- if segment_reader.is_deleted(doc_id) {- doc_id = segment_postings.advance();- continue;- }-- // the number of time the term appears in the document.- let term_freq: u32 = segment_postings.term_freq();- // accessing positions is slightly expensive and lazy, do not request- // for them if you don't need them for some documents.- segment_postings.positions(&mut positions);-- // By definition we should have `term_freq` positions.- assert_eq!(positions.len(), term_freq as usize);-- // This prints:- // ```- // Doc 0: TermFreq 2: [0, 4]- // Doc 2: TermFreq 1: [0]- // ```- println!("Doc {doc_id}: TermFreq {term_freq}: {positions:?}");- doc_id = segment_postings.advance();- }- }- }-- // A `Term` is a text token associated with a field.- // Let's go through all docs containing the term `title:the` and access their position- let term_the = Term::from_field_text(title, "the");-- // Some other powerful operations (especially `.skip_to`) may be useful to consume these- // posting lists rapidly.- // You can check for them in the [`DocSet`](https://docs.rs/tantivy/~0/tantivy/trait.DocSet.html) trait- // and the [`Postings`](https://docs.rs/tantivy/~0/tantivy/trait.Postings.html) trait-- // Also, for some VERY specific high performance use case like an OLAP analysis of logs,- // you can get better performance by accessing directly the blocks of doc ids.- for segment_reader in searcher.segment_readers() {- // A segment contains different data structure.- // Inverted index stands for the combination of- // - the term dictionary- // - the inverted lists associated with each terms and their positions- let inverted_index = segment_reader.inverted_index(title)?;-- // This segment posting object is like a cursor over the documents matching the term.- // The `IndexRecordOption` arguments tells tantivy we will be interested in both term- // frequencies and positions.- //- // If you don't need all this information, you may get better performance by decompressing- // less information.- if let Some(mut block_segment_postings) =- inverted_index.read_block_postings(&term_the, IndexRecordOption::Basic)?- {- loop {- let docs = block_segment_postings.docs();- if docs.is_empty() {- break;- }- // Once again these docs MAY contains deleted documents as well.- let docs = block_segment_postings.docs();- // Prints `Docs [0, 2].`- println!("Docs {docs:?}");- block_segment_postings.advance();- }- }- }-- Ok(())-}diff --git a/crates/tantivy/examples/json_field.rs b/crates/tantivy/examples/json_field.rsdeleted file mode 100644index 334ea10e..00000000--- a/crates/tantivy/examples/json_field.rs+++ /dev/null@@ -1,107 +0,0 @@-// # Json field example-//-// This example shows how the json field can be used-// to make tantivy partially schemaless by setting it as-// default query parser field.--use tantivy::collector::{Count, TopDocs};-use tantivy::query::QueryParser;-use tantivy::schema::{Schema, FAST, STORED, STRING, TEXT};-use tantivy::{Index, IndexWriter, TantivyDocument};--fn main() -> tantivy::Result<()> {- // # Defining the schema- let mut schema_builder = Schema::builder();- schema_builder.add_date_field("timestamp", FAST | STORED);- let event_type = schema_builder.add_text_field("event_type", STRING | STORED);- let attributes = schema_builder.add_json_field("attributes", STORED | TEXT);- let schema = schema_builder.build();-- // # Indexing documents- let index = Index::create_in_ram(schema.clone());-- let mut index_writer: IndexWriter = index.writer(50_000_000)?;- let doc = TantivyDocument::parse_json(- &schema,- r#"{- "timestamp": "2022-02-22T23:20:50.53Z",- "event_type": "click",- "attributes": {- "target": "submit-button",- "cart": {"product_id": 103},- "description": "the best vacuum cleaner ever"- }- }"#,- )?;- index_writer.add_document(doc)?;- let doc = TantivyDocument::parse_json(- &schema,- r#"{- "timestamp": "2022-02-22T23:20:51.53Z",- "event_type": "click",- "attributes": {- "target": "submit-button",- "cart": {"product_id": 133},- "description": "das keyboard",- "event_type": "holiday-sale"- }- }"#,- )?;- index_writer.add_document(doc)?;- index_writer.commit()?;-- let reader = index.reader()?;- let searcher = reader.searcher();-- // # Default fields: event_type and attributes- // By setting attributes as a default field it allows omitting attributes itself, e.g. "target",- // instead of "attributes.target"- let query_parser = QueryParser::for_index(&index, vec![event_type, attributes]);- {- let query = query_parser.parse_query("target:submit-button")?;- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2))?;- assert_eq!(count_docs.len(), 2);- }- {- let query = query_parser.parse_query("target:submit")?;- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2))?;- assert_eq!(count_docs.len(), 2);- }- {- let query = query_parser.parse_query("cart.product_id:103")?;- let count_docs = searcher.search(&*query, &Count)?;- assert_eq!(count_docs, 1);- }- {- let query = query_parser.parse_query("click AND cart.product_id:133")?;- let hits = searcher.search(&*query, &TopDocs::with_limit(2))?;- assert_eq!(hits.len(), 1);- }- {- // The sub-fields in the json field marked as default field still need to be explicitly- // addressed- let query = query_parser.parse_query("click AND 133")?;- let hits = searcher.search(&*query, &TopDocs::with_limit(2))?;- assert_eq!(hits.len(), 0);- }- {- // Default json fields are ignored if they collide with the schema- let query = query_parser.parse_query("event_type:holiday-sale")?;- let hits = searcher.search(&*query, &TopDocs::with_limit(2))?;- assert_eq!(hits.len(), 0);- }- // # Query via full attribute path- {- // This only searches in our schema's `event_type` field- let query = query_parser.parse_query("event_type:click")?;- let hits = searcher.search(&*query, &TopDocs::with_limit(2))?;- assert_eq!(hits.len(), 2);- }- {- // Default json fields can still be accessed by full path- let query = query_parser.parse_query("attributes.event_type:holiday-sale")?;- let hits = searcher.search(&*query, &TopDocs::with_limit(2))?;- assert_eq!(hits.len(), 1);- }- Ok(())-}diff --git a/crates/tantivy/examples/phrase_prefix_search.rs b/crates/tantivy/examples/phrase_prefix_search.rsdeleted file mode 100644index 24239dc7..00000000--- a/crates/tantivy/examples/phrase_prefix_search.rs+++ /dev/null@@ -1,83 +0,0 @@-use tantivy::collector::TopDocs;-use tantivy::query::QueryParser;-use tantivy::schema::*;-use tantivy::{doc, Index, IndexWriter, ReloadPolicy, Result};-use tempfile::TempDir;--fn main() -> Result<()> {- let index_path = TempDir::new()?;-- let mut schema_builder = Schema::builder();- schema_builder.add_text_field("title", TEXT | STORED);- schema_builder.add_text_field("body", TEXT);- let schema = schema_builder.build();-- let title = schema.get_field("title").unwrap();- let body = schema.get_field("body").unwrap();-- let index = Index::create_in_dir(&index_path, schema)?;-- let mut index_writer: IndexWriter = index.writer(50_000_000)?;-- index_writer.add_document(doc!(- title => "The Old Man and the Sea",- body => "He was an old man who fished alone in a skiff in the Gulf Stream and he had gone \- eighty-four days now without taking a fish.",- ))?;-- index_writer.add_document(doc!(- title => "Of Mice and Men",- body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \- bank and runs deep and green. The water is warm too, for it has slipped twinkling \- over the yellow sands in the sunlight before reaching the narrow pool. On one \- side of the river the golden foothill slopes curve up to the strong and rocky \- Gabilan Mountains, but on the valley side the water is lined with trees—willows \- fresh and green with every spring, carrying in their lower leaf junctures the \- debris of the winter’s flooding; and sycamores with mottled, white, recumbent \- limbs and branches that arch over the pool"- ))?;-- // Multivalued field just need to be repeated.- index_writer.add_document(doc!(- title => "Frankenstein",- title => "The Modern Prometheus",- body => "You will rejoice to hear that no disaster has accompanied the commencement of an \- enterprise which you have regarded with such evil forebodings. I arrived here \- yesterday, and my first task is to assure my dear sister of my welfare and \- increasing confidence in the success of my undertaking."- ))?;-- index_writer.commit()?;-- let reader = index- .reader_builder()- .reload_policy(ReloadPolicy::OnCommitWithDelay)- .try_into()?;-- let searcher = reader.searcher();-- let query_parser = QueryParser::for_index(&index, vec![title, body]);- // This will match documents containing the phrase "in the"- // followed by some word starting with "su",- // i.e. it will match "in the sunlight" and "in the success",- // but not "in the Gulf Stream".- let query = query_parser.parse_query("\"in the su\"*")?;-- let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;- let mut titles = top_docs- .into_iter()- .map(|(_score, doc_address)| {- let doc = searcher.doc::<TantivyDocument>(doc_address)?;- let title = doc- .get_first(title)- .and_then(|v| v.as_str())- .unwrap()- .to_owned();- Ok(title)- })- .collect::<Result<Vec<_>>>()?;- titles.sort_unstable();- assert_eq!(titles, ["Frankenstein", "Of Mice and Men"]);-- Ok(())-}diff --git a/crates/tantivy/examples/pre_tokenized_text.rs b/crates/tantivy/examples/pre_tokenized_text.rsdeleted file mode 100644index 4cd4e930..00000000--- a/crates/tantivy/examples/pre_tokenized_text.rs+++ /dev/null@@ -1,136 +0,0 @@-// # Pre-tokenized text example-//-// This example shows how to use pre-tokenized text. Sometimes you might-// want to index and search through text which is already split into-// tokens by some external tool.-//-// In this example we will:-// - use tantivy tokenizer to create tokens and load them directly into tantivy,-// - import tokenized text straight from json,-// - perform a search on documents with pre-tokenized text--use tantivy::collector::{Count, TopDocs};-use tantivy::query::TermQuery;-use tantivy::schema::*;-use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, TokenStream, Tokenizer};-use tantivy::{doc, Index, IndexWriter, ReloadPolicy};-use tempfile::TempDir;--fn pre_tokenize_text(text: &str) -> Vec<Token> {- let mut tokenizer = SimpleTokenizer::default();- let mut token_stream = tokenizer.token_stream(text);- let mut tokens = vec![];- while token_stream.advance() {- tokens.push(token_stream.token().clone());- }- tokens-}--fn main() -> tantivy::Result<()> {- let index_path = TempDir::new()?;-- let mut schema_builder = Schema::builder();-- schema_builder.add_text_field("title", TEXT | STORED);- schema_builder.add_text_field("body", TEXT);-- let schema = schema_builder.build();-- let index = Index::create_in_dir(&index_path, schema.clone())?;-- let mut index_writer: IndexWriter = index.writer(50_000_000)?;-- // We can create a document manually, by setting the fields- // one by one in a Document object.- let title = schema.get_field("title").unwrap();- let body = schema.get_field("body").unwrap();-- let title_text = "The Old Man and the Sea";- let body_text = "He was an old man who fished alone in a skiff in the Gulf Stream";-- // Content of our first document- // We create `PreTokenizedString` which contains original text and vector of tokens- let title_tok = PreTokenizedString {- text: String::from(title_text),- tokens: pre_tokenize_text(title_text),- };-- println!(- "Original text: \"{}\" and tokens: {:?}",- title_tok.text, title_tok.tokens- );-- let body_tok = PreTokenizedString {- text: String::from(body_text),- tokens: pre_tokenize_text(body_text),- };-- // Now lets create a document and add our `PreTokenizedString`- let old_man_doc = doc!(title => title_tok, body => body_tok);-- // ... now let's just add it to the IndexWriter- index_writer.add_document(old_man_doc)?;-- // Pretokenized text can also be fed as JSON- let short_man_json = r#"{- "title":[{- "text":"The Old Man",- "tokens":[- {"offset_from":0,"offset_to":3,"position":0,"text":"The","position_length":1},- {"offset_from":4,"offset_to":7,"position":1,"text":"Old","position_length":1},- {"offset_from":8,"offset_to":11,"position":2,"text":"Man","position_length":1}- ]- }]- }"#;-- let short_man_doc = TantivyDocument::parse_json(&schema, short_man_json)?;-- index_writer.add_document(short_man_doc)?;-- // Let's commit changes- index_writer.commit()?;-- // ... and now is the time to query our index-- let reader = index- .reader_builder()- .reload_policy(ReloadPolicy::OnCommitWithDelay)- .try_into()?;-- let searcher = reader.searcher();-- // We want to get documents with token "Man", we will use TermQuery to do it- // Using PreTokenizedString means the tokens are stored as is avoiding stemming- // and lowercasing, which preserves full words in their original form- let query = TermQuery::new(- Term::from_field_text(title, "Man"),- IndexRecordOption::Basic,- );-- let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count))?;-- assert_eq!(count, 2);-- // Now let's print out the results.- // Note that the tokens are not stored along with the original text- // in the document store- for (_score, doc_address) in top_docs {- let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;- println!("{}", retrieved_doc.to_json(&schema));- }-- // In contrary to the previous query, when we search for the "man" term we- // should get no results, as it's not one of the indexed tokens. SimpleTokenizer- // only splits text on whitespace / punctuation.-- let query = TermQuery::new(- Term::from_field_text(title, "man"),- IndexRecordOption::Basic,- );-- let (_top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count))?;-- assert_eq!(count, 0);-- Ok(())-}diff --git a/crates/tantivy/examples/snippet.rs b/crates/tantivy/examples/snippet.rsdeleted file mode 100644index 31bd2c16..00000000--- a/crates/tantivy/examples/snippet.rs+++ /dev/null@@ -1,83 +0,0 @@-// # Snippet example-//-// This example shows how to return a representative snippet of-// your hit result.-// Snippet are an extracted of a target document, and returned in HTML format.-// The keyword searched by the user are highlighted with a `<b>` tag.--// ----// Importing tantivy...-use tantivy::collector::TopDocs;-use tantivy::query::QueryParser;-use tantivy::schema::*;-use tantivy::snippet::{Snippet, SnippetGenerator};-use tantivy::{doc, Index, IndexWriter};-use tempfile::TempDir;--fn main() -> tantivy::Result<()> {- // Let's create a temporary directory for the- // sake of this example- let index_path = TempDir::new()?;-- // # Defining the schema- let mut schema_builder = Schema::builder();- let title = schema_builder.add_text_field("title", TEXT | STORED);- let body = schema_builder.add_text_field("body", TEXT | STORED);- let schema = schema_builder.build();-- // # Indexing documents- let index = Index::create_in_dir(&index_path, schema)?;-- let mut index_writer: IndexWriter = index.writer(50_000_000)?;-- // we'll only need one doc for this example.- index_writer.add_document(doc!(- title => "Of Mice and Men",- body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \- bank and runs deep and green. The water is warm too, for it has slipped twinkling \- over the yellow sands in the sunlight before reaching the narrow pool. On one \- side of the river the golden foothill slopes curve up to the strong and rocky \- Gabilan Mountains, but on the valley side the water is lined with trees—willows \- fresh and green with every spring, carrying in their lower leaf junctures the \- debris of the winter’s flooding; and sycamores with mottled, white, recumbent \- limbs and branches that arch over the pool"- ))?;- // ...- index_writer.commit()?;-- let reader = index.reader()?;- let searcher = reader.searcher();- let query_parser = QueryParser::for_index(&index, vec![title, body]);- let query = query_parser.parse_query("sycamore spring")?;-- let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;-- let snippet_generator = SnippetGenerator::create(&searcher, &*query, body)?;-- for (score, doc_address) in top_docs {- let doc = searcher.doc::<TantivyDocument>(doc_address)?;- let snippet = snippet_generator.snippet_from_doc(&doc);- println!("Document score {score}:");- println!("title: {}", doc.get_first(title).unwrap().as_str().unwrap());- println!("snippet: {}", snippet.to_html());- println!("custom highlighting: {}", highlight(snippet));- }-- Ok(())-}--fn highlight(snippet: Snippet) -> String {- let mut result = String::new();- let mut start_from = 0;-- for fragment_range in snippet.highlighted() {- result.push_str(&snippet.fragment()[start_from..fragment_range.start]);- result.push_str(" --> ");- result.push_str(&snippet.fragment()[fragment_range.clone()]);- result.push_str(" <-- ");- start_from = fragment_range.end;- }-- result.push_str(&snippet.fragment()[start_from..]);- result-}diff --git a/crates/tantivy/examples/stop_words.rs b/crates/tantivy/examples/stop_words.rsdeleted file mode 100644index 80dab3fe..00000000--- a/crates/tantivy/examples/stop_words.rs+++ /dev/null@@ -1,114 +0,0 @@-// # Stop Words Example-//-// This example covers the basic usage of stop words-// with tantivy-//-// We will :-// - define our schema-// - create an index in a directory-// - add a few stop words-// - index few documents in our index--// ----// Importing tantivy...-use tantivy::collector::TopDocs;-use tantivy::query::QueryParser;-use tantivy::schema::*;-use tantivy::tokenizer::*;-use tantivy::{doc, Index, IndexWriter};--fn main() -> tantivy::Result<()> {- // this example assumes you understand the content in `basic_search`- let mut schema_builder = Schema::builder();-- // This configures your custom options for how tantivy will- // store and process your content in the index; The key- // to note is that we are setting the tokenizer to `stoppy`- // which will be defined and registered below.- let text_field_indexing = TextFieldIndexing::default()- .set_tokenizer("stoppy")- .set_index_option(IndexRecordOption::WithFreqsAndPositions);- let text_options = TextOptions::default()- .set_indexing_options(text_field_indexing)- .set_stored();-- // Our first field is title.- schema_builder.add_text_field("title", text_options);-- // Our second field is body.- let text_field_indexing = TextFieldIndexing::default()- .set_tokenizer("stoppy")- .set_index_option(IndexRecordOption::WithFreqsAndPositions);- let text_options = TextOptions::default()- .set_indexing_options(text_field_indexing)- .set_stored();- schema_builder.add_text_field("body", text_options);-- let schema = schema_builder.build();-- let index = Index::create_in_ram(schema.clone());-- // This tokenizer lowers all of the text (to help with stop word matching)- // then removes all instances of `the` and `and` from the corpus- let tokenizer = TextAnalyzer::builder(SimpleTokenizer::default())- .filter(LowerCaser)- .filter(StopWordFilter::remove(vec![- "the".to_string(),- "and".to_string(),- ]))- .build();-- index.tokenizers().register("stoppy", tokenizer);-- let mut index_writer: IndexWriter = index.writer(50_000_000)?;-- let title = schema.get_field("title").unwrap();- let body = schema.get_field("body").unwrap();-- index_writer.add_document(doc!(- title => "The Old Man and the Sea",- body => "He was an old man who fished alone in a skiff in the Gulf Stream and \- he had gone eighty-four days now without taking a fish."- ))?;-- index_writer.add_document(doc!(- title => "Of Mice and Men",- body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \- bank and runs deep and green. The water is warm too, for it has slipped twinkling \- over the yellow sands in the sunlight before reaching the narrow pool. On one \- side of the river the golden foothill slopes curve up to the strong and rocky \- Gabilan Mountains, but on the valley side the water is lined with trees—willows \- fresh and green with every spring, carrying in their lower leaf junctures the \- debris of the winter’s flooding; and sycamores with mottled, white, recumbent \- limbs and branches that arch over the pool"- ))?;-- index_writer.add_document(doc!(- title => "Frankenstein",- body => "You will rejoice to hear that no disaster has accompanied the commencement of an \- enterprise which you have regarded with such evil forebodings. I arrived here \- yesterday, and my first task is to assure my dear sister of my welfare and \- increasing confidence in the success of my undertaking."- ))?;-- index_writer.commit()?;-- let reader = index.reader()?;-- let searcher = reader.searcher();-- let query_parser = QueryParser::for_index(&index, vec![title, body]);-- // stop words are applied on the query as well.- // The following will be equivalent to `title:frankenstein`- let query = query_parser.parse_query("title:\"the Frankenstein\"")?;- let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;-- for (score, doc_address) in top_docs {- let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;- println!("\n==\nDocument score {score}:");- println!("{}", retrieved_doc.to_json(&schema));- }-- Ok(())-}diff --git a/crates/tantivy/examples/warmer.rs b/crates/tantivy/examples/warmer.rsdeleted file mode 100644index 1cae9d34..00000000--- a/crates/tantivy/examples/warmer.rs+++ /dev/null@@ -1,225 +0,0 @@-use std::cmp::Reverse;-use std::collections::{HashMap, HashSet};-use std::sync::{Arc, RwLock, Weak};--use tantivy::collector::TopDocs;-use tantivy::index::SegmentId;-use tantivy::query::QueryParser;-use tantivy::schema::{Schema, FAST, TEXT};-use tantivy::{- doc, DocAddress, DocId, Index, IndexWriter, Opstamp, Searcher, SearcherGeneration,- SegmentReader, Warmer,-};--// This example shows how warmers can be used to-// load values from an external sources and-// tie their lifecycle to that of the index segments-// using the Warmer API.-//-// In this example, we assume an e-commerce search engine.--type ProductId = u64;--type Price = u32;--pub trait PriceFetcher: Send + Sync + 'static {- fn fetch_prices(&self, product_ids: &[ProductId]) -> Vec<Price>;-}--type SegmentKey = (SegmentId, Option<Opstamp>);--struct DynamicPriceColumn {- field: String,- price_cache: RwLock<HashMap<SegmentKey, Arc<Vec<Price>>>>,- price_fetcher: Box<dyn PriceFetcher>,-}--impl DynamicPriceColumn {- pub fn with_product_id_field<T: PriceFetcher>(field: String, price_fetcher: T) -> Self {- DynamicPriceColumn {- field,- price_cache: Default::default(),- price_fetcher: Box::new(price_fetcher),- }- }-- pub fn price_for_segment(&self, segment_reader: &SegmentReader) -> Option<Arc<Vec<Price>>> {- let segment_key = (segment_reader.segment_id(), segment_reader.delete_opstamp());- self.price_cache.read().unwrap().get(&segment_key).cloned()- }-}-impl Warmer for DynamicPriceColumn {- fn warm(&self, searcher: &Searcher) -> tantivy::Result<()> {- for segment in searcher.segment_readers() {- let product_id_reader = segment- .fast_fields()- .u64(&self.field)?- .first_or_default_col(0);- let product_ids: Vec<ProductId> = segment- .doc_ids_alive()- .map(|doc| product_id_reader.get_val(doc))- .collect();-- let mut prices = self.price_fetcher.fetch_prices(&product_ids).into_iter();-- let prices: Vec<Price> = (0..segment.max_doc())- .map(|doc| {- if !segment.is_deleted(doc) {- prices.next().unwrap()- } else {- 0- }- })- .collect();-- let key = (segment.segment_id(), segment.delete_opstamp());- self.price_cache- .write()- .unwrap()- .insert(key, Arc::new(prices));- }-- Ok(())- }-- fn garbage_collect(&self, live_generations: &[&SearcherGeneration]) {- let live_keys: HashSet<SegmentKey> = live_generations- .iter()- .flat_map(|gen| gen.segments())- .map(|(&segment_id, &opstamp)| (segment_id, opstamp))- .collect();-- self.price_cache- .write()- .unwrap()- .retain(|key, _| live_keys.contains(key));- }-}--// For the sake of this example, the table is just an editable HashMap behind a RwLock.-// This map represents a map (ProductId -> Price)-//-// In practise, it could be fetching things from an external service, like a SQL table.-#[derive(Default, Clone)]-pub struct ExternalPriceTable {- prices: Arc<RwLock<HashMap<ProductId, Price>>>,-}--impl ExternalPriceTable {- pub fn update_price(&self, product_id: ProductId, price: Price) {- self.prices.write().unwrap().insert(product_id, price);- }-}--impl PriceFetcher for ExternalPriceTable {- fn fetch_prices(&self, product_ids: &[ProductId]) -> Vec<Price> {- let prices = self.prices.read().unwrap();-- product_ids- .iter()- .map(|product_id| prices.get(product_id).cloned().unwrap_or(0))- .collect()- }-}--fn main() -> tantivy::Result<()> {- // Declaring our schema.- let mut schema_builder = Schema::builder();- // The product id is assumed to be a primary id for our external price source.- let product_id = schema_builder.add_u64_field("product_id", FAST);- let text = schema_builder.add_text_field("text", TEXT);- let schema: Schema = schema_builder.build();-- let price_table = ExternalPriceTable::default();- let price_dynamic_column = Arc::new(DynamicPriceColumn::with_product_id_field(- "product_id".to_string(),- price_table.clone(),- ));- price_table.update_price(OLIVE_OIL, 12);- price_table.update_price(GLOVES, 13);- price_table.update_price(SNEAKERS, 80);-- const OLIVE_OIL: ProductId = 323423;- const GLOVES: ProductId = 3966623;- const SNEAKERS: ProductId = 23222;-- let index = Index::create_in_ram(schema);- let mut writer: IndexWriter = index.writer_with_num_threads(1, 15_000_000)?;- writer.add_document(doc!(product_id=>OLIVE_OIL, text=>"cooking olive oil from greece"))?;- writer.add_document(doc!(product_id=>GLOVES, text=>"kitchen gloves, perfect for cooking"))?;- writer.add_document(doc!(product_id=>SNEAKERS, text=>"uber sweet sneakers"))?;- writer.commit()?;-- let warmers = vec![Arc::downgrade(&price_dynamic_column) as Weak<dyn Warmer>];- let reader = index.reader_builder().warmers(warmers).try_into()?;-- let query_parser = QueryParser::for_index(&index, vec![text]);- let query = query_parser.parse_query("cooking")?;-- let searcher = reader.searcher();- let score_by_price = move |segment_reader: &SegmentReader| {- let price = price_dynamic_column- .price_for_segment(segment_reader)- .unwrap();- move |doc_id: DocId| Reverse(price[doc_id as usize])- };-- let most_expensive_first = TopDocs::with_limit(10).custom_score(score_by_price);-- let hits = searcher.search(&query, &most_expensive_first)?;- assert_eq!(- &hits,- &[- (- Reverse(12u32),- DocAddress {- segment_ord: 0,- doc_id: 0u32- }- ),- (- Reverse(13u32),- DocAddress {- segment_ord: 0,- doc_id: 1u32- }- ),- ]- );-- // Olive oil just got more expensive!- price_table.update_price(OLIVE_OIL, 15);-- // The price update are directly reflected on `reload`.- //- // Be careful here though!...- // You may have spotted that we are still using the same `Searcher`.- //- // It is up to the `Warmer` implementer to decide how- // to control this behavior.-- reader.reload()?;-- let hits_with_new_prices = searcher.search(&query, &most_expensive_first)?;- assert_eq!(- &hits_with_new_prices,- &[- (- Reverse(13u32),- DocAddress {- segment_ord: 0,- doc_id: 1u32- }- ),- (- Reverse(15u32),- DocAddress {- segment_ord: 0,- doc_id: 0u32- }- ),- ]- );-- Ok(())-}diff --git a/crates/tantivy/src/collector/count_collector.rs b/crates/tantivy/src/collector/count_collector.rsindex dcd10224..60c73bd2 100644--- a/crates/tantivy/src/collector/count_collector.rs+++ b/crates/tantivy/src/collector/count_collector.rs@@ -4,35 +4,6 @@ use crate::{DocId, Score, SegmentOrdinal, SegmentReader};/// `CountCollector` collector only counts how many/// documents match the query.-///-/// ```rust-/// use tantivy::collector::Count;-/// use tantivy::query::QueryParser;-/// use tantivy::schema::{Schema, TEXT};-/// use tantivy::{doc, Index};-///-/// let mut schema_builder = Schema::builder();-/// let title = schema_builder.add_text_field("title", TEXT);-/// let schema = schema_builder.build();-/// let index = Index::create_in_ram(schema);-///-/// let mut index_writer = index.writer(15_000_000).unwrap();-/// index_writer.add_document(doc!(title => "The Name of the Wind")).unwrap();-/// index_writer.add_document(doc!(title => "The Diary of Muadib")).unwrap();-/// index_writer.add_document(doc!(title => "A Dairy Cow")).unwrap();-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl")).unwrap();-/// assert!(index_writer.commit().is_ok());-///-/// let reader = index.reader().unwrap();-/// let searcher = reader.searcher();-///-/// // Here comes the important part-/// let query_parser = QueryParser::for_index(&index, vec![title]);-/// let query = query_parser.parse_query("diary").unwrap();-/// let count = searcher.search(&query, &Count).unwrap();-///-/// assert_eq!(count, 2);-/// ```pub struct Count;impl Collector for Count {diff --git a/crates/tantivy/src/collector/filter_collector_wrapper.rs b/crates/tantivy/src/collector/filter_collector_wrapper.rsindex 259ec49d..df121222 100644--- a/crates/tantivy/src/collector/filter_collector_wrapper.rs+++ b/crates/tantivy/src/collector/filter_collector_wrapper.rs@@ -23,46 +23,6 @@ use crate::{DocId, Score, SegmentReader};/// - documents with no values are filtered out./// - documents with several values are accepted if at least one value matches the predicate.///-///-/// ```rust-/// use tantivy::collector::{TopDocs, FilterCollector};-/// use tantivy::query::QueryParser;-/// use tantivy::schema::{Schema, TEXT, FAST};-/// use tantivy::{doc, DocAddress, Index};-///-/// # fn main() -> tantivy::Result<()> {-/// let mut schema_builder = Schema::builder();-/// let title = schema_builder.add_text_field("title", TEXT);-/// let price = schema_builder.add_u64_field("price", FAST);-/// let schema = schema_builder.build();-/// let index = Index::create_in_ram(schema);-///-/// let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;-/// index_writer.add_document(doc!(title => "The Name of the Wind", price => 30_200u64))?;-/// index_writer.add_document(doc!(title => "The Diary of Muadib", price => 29_240u64))?;-/// index_writer.add_document(doc!(title => "A Dairy Cow", price => 21_240u64))?;-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl", price => 20_120u64))?;-/// index_writer.commit()?;-///-/// let reader = index.reader()?;-/// let searcher = reader.searcher();-///-/// let query_parser = QueryParser::for_index(&index, vec![title]);-/// let query = query_parser.parse_query("diary")?;-/// let no_filter_collector = FilterCollector::new("price".to_string(), |value: u64| value > 20_120u64, TopDocs::with_limit(2));-/// let top_docs = searcher.search(&query, &no_filter_collector)?;-///-/// assert_eq!(top_docs.len(), 1);-/// assert_eq!(top_docs[0].1, DocAddress::new(0, 1));-///-/// let filter_all_collector: FilterCollector<_, _, u64> = FilterCollector::new("price".to_string(), |value| value < 5u64, TopDocs::with_limit(2));-/// let filtered_top_docs = searcher.search(&query, &filter_all_collector)?;-///-/// assert_eq!(filtered_top_docs.len(), 0);-/// # Ok(())-/// # }-/// ```-////// Note that this is limited to fast fields which implement the/// [`FastValue`][crate::fastfield::FastValue] trait, e.g. `u64` but not `&[u8]`./// To filter based on a bytes fast field, use a [`BytesFilterCollector`] instead.@@ -191,41 +151,6 @@ where/// In other words,/// - documents with no values are filtered out./// - documents with several values are accepted if at least one value matches the predicate.-///-/// ```rust-/// use tantivy::collector::{TopDocs, BytesFilterCollector};-/// use tantivy::query::QueryParser;-/// use tantivy::schema::{Schema, TEXT, FAST};-/// use tantivy::{doc, DocAddress, Index};-///-/// # fn main() -> tantivy::Result<()> {-/// let mut schema_builder = Schema::builder();-/// let title = schema_builder.add_text_field("title", TEXT);-/// let barcode = schema_builder.add_bytes_field("barcode", FAST);-/// let schema = schema_builder.build();-/// let index = Index::create_in_ram(schema);-///-/// let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;-/// index_writer.add_document(doc!(title => "The Name of the Wind", barcode => &b"010101"[..]))?;-/// index_writer.add_document(doc!(title => "The Diary of Muadib", barcode => &b"110011"[..]))?;-/// index_writer.add_document(doc!(title => "A Dairy Cow", barcode => &b"110111"[..]))?;-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl", barcode => &b"011101"[..]))?;-/// index_writer.add_document(doc!(title => "Bridget Jones's Diary"))?;-/// index_writer.commit()?;-///-/// let reader = index.reader()?;-/// let searcher = reader.searcher();-///-/// let query_parser = QueryParser::for_index(&index, vec![title]);-/// let query = query_parser.parse_query("diary")?;-/// let filter_collector = BytesFilterCollector::new("barcode".to_string(), |bytes: &[u8]| bytes.starts_with(b"01"), TopDocs::with_limit(2));-/// let top_docs = searcher.search(&query, &filter_collector)?;-///-/// assert_eq!(top_docs.len(), 1);-/// assert_eq!(top_docs[0].1, DocAddress::new(0, 3));-/// # Ok(())-/// # }-/// ```pub struct BytesFilterCollector<TCollector, TPredicate>whereTPredicate: 'static + Clone,diff --git a/crates/tantivy/src/collector/mod.rs b/crates/tantivy/src/collector/mod.rsindex b78e0207..1325f43e 100644--- a/crates/tantivy/src/collector/mod.rs+++ b/crates/tantivy/src/collector/mod.rs@@ -33,35 +33,6 @@//! The resulting `Fruit` will then be a typed tuple with each collector's original fruits//! in their respective position.//!-//! ```rust-//! # use tantivy::schema::*;-//! # use tantivy::*;-//! # use tantivy::query::*;-//! use tantivy::collector::{Count, TopDocs};-//! #-//! # fn main() -> tantivy::Result<()> {-//! # let mut schema_builder = Schema::builder();-//! # let title = schema_builder.add_text_field("title", TEXT);-//! # let schema = schema_builder.build();-//! # let index = Index::create_in_ram(schema);-//! # let mut index_writer = index.writer(15_000_000)?;-//! # index_writer.add_document(doc!(-//! # title => "The Name of the Wind",-//! # ))?;-//! # index_writer.add_document(doc!(-//! # title => "The Diary of Muadib",-//! # ))?;-//! # index_writer.commit()?;-//! # let reader = index.reader()?;-//! # let searcher = reader.searcher();-//! # let query_parser = QueryParser::for_index(&index, vec![title]);-//! # let query = query_parser.parse_query("diary")?;-//! let (doc_count, top_docs): (usize, Vec<(Score, DocAddress)>) =-//! searcher.search(&query, &(Count, TopDocs::with_limit(2)))?;-//! # Ok(())-//! # }-//! ```-//!//! The `Collector` trait is implemented for up to 4 collectors.//! If you have more than 4 collectors, you can either group them into//! tuples of tuples `(a,(b,(c,d)))`, or rely on [`MultiCollector`].diff --git a/crates/tantivy/src/collector/multi_collector.rs b/crates/tantivy/src/collector/multi_collector.rsindex da4d222a..7acf745d 100644--- a/crates/tantivy/src/collector/multi_collector.rs+++ b/crates/tantivy/src/collector/multi_collector.rs@@ -124,43 +124,6 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {/// in a tuple. See the/// [Combining several collectors section of the collector/// documentation](./index.html#combining-several-collectors).-///-/// ```rust-/// use tantivy::collector::{Count, TopDocs, MultiCollector};-/// use tantivy::query::QueryParser;-/// use tantivy::schema::{Schema, TEXT};-/// use tantivy::{doc, Index};-///-/// # fn main() -> tantivy::Result<()> {-/// let mut schema_builder = Schema::builder();-/// let title = schema_builder.add_text_field("title", TEXT);-/// let schema = schema_builder.build();-/// let index = Index::create_in_ram(schema);-/// let mut index_writer = index.writer(15_000_000)?;-/// index_writer.add_document(doc!(title => "The Name of the Wind"))?;-/// index_writer.add_document(doc!(title => "The Diary of Muadib"))?;-/// index_writer.add_document(doc!(title => "A Dairy Cow"))?;-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"))?;-/// index_writer.commit()?;-///-/// let reader = index.reader()?;-/// let searcher = reader.searcher();-///-/// let mut collectors = MultiCollector::new();-/// let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));-/// let count_handle = collectors.add_collector(Count);-/// let query_parser = QueryParser::for_index(&index, vec![title]);-/// let query = query_parser.parse_query("diary").unwrap();-/// let mut multi_fruit = searcher.search(&query, &collectors).unwrap();-///-/// let count = count_handle.extract(&mut multi_fruit);-/// let top_docs = top_docs_handle.extract(&mut multi_fruit);-///-/// assert_eq!(count, 2);-/// assert_eq!(top_docs.len(), 2);-/// # Ok(())-/// # }-/// ```#[allow(clippy::type_complexity)]#[derive(Default)]pub struct MultiCollector<'a> {diff --git a/crates/tantivy/src/collector/top_score_collector.rs b/crates/tantivy/src/collector/top_score_collector.rsindex 305050c7..a3eb05c1 100644--- a/crates/tantivy/src/collector/top_score_collector.rs+++ b/crates/tantivy/src/collector/top_score_collector.rs@@ -96,38 +96,6 @@ where/// document score, for stable sorting `PartialOrd` needs to resolve on other fields/// like docid in case of score equality./// Only then, it is suitable for pagination.-///-/// ```rust-/// use tantivy::collector::TopDocs;-/// use tantivy::query::QueryParser;-/// use tantivy::schema::{Schema, TEXT};-/// use tantivy::{doc, DocAddress, Index};-///-/// # fn main() -> tantivy::Result<()> {-/// let mut schema_builder = Schema::builder();-/// let title = schema_builder.add_text_field("title", TEXT);-/// let schema = schema_builder.build();-/// let index = Index::create_in_ram(schema);-///-/// let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;-/// index_writer.add_document(doc!(title => "The Name of the Wind"))?;-/// index_writer.add_document(doc!(title => "The Diary of Muadib"))?;-/// index_writer.add_document(doc!(title => "A Dairy Cow"))?;-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"))?;-/// index_writer.commit()?;-///-/// let reader = index.reader()?;-/// let searcher = reader.searcher();-///-/// let query_parser = QueryParser::for_index(&index, vec![title]);-/// let query = query_parser.parse_query("diary")?;-/// let top_docs = searcher.search(&query, &TopDocs::with_limit(2))?;-///-/// assert_eq!(top_docs[0].1, DocAddress::new(0, 1));-/// assert_eq!(top_docs[1].1, DocAddress::new(0, 3));-/// # Ok(())-/// # }-/// ```pub struct TopDocs(TopCollector<Score>);impl fmt::Debug for TopDocs {@@ -201,40 +169,6 @@ impl TopDocs {/// Lucene's TopDocsCollector.////// # Example- ///- /// ```rust- /// use tantivy::collector::TopDocs;- /// use tantivy::query::QueryParser;- /// use tantivy::schema::{Schema, TEXT};- /// use tantivy::{doc, DocAddress, Index};- ///- /// # fn main() -> tantivy::Result<()> {- /// let mut schema_builder = Schema::builder();- /// let title = schema_builder.add_text_field("title", TEXT);- /// let schema = schema_builder.build();- /// let index = Index::create_in_ram(schema);- ///- /// let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;- /// index_writer.add_document(doc!(title => "The Name of the Wind"))?;- /// index_writer.add_document(doc!(title => "The Diary of Muadib"))?;- /// index_writer.add_document(doc!(title => "A Dairy Cow"))?;- /// index_writer.add_document(doc!(title => "The Diary of a Young Girl"))?;- /// index_writer.add_document(doc!(title => "The Diary of Lena Mukhina"))?;- /// index_writer.commit()?;- ///- /// let reader = index.reader()?;- /// let searcher = reader.searcher();- ///- /// let query_parser = QueryParser::for_index(&index, vec![title]);- /// let query = query_parser.parse_query("diary")?;- /// let top_docs = searcher.search(&query, &TopDocs::with_limit(2).and_offset(1))?;- ///- /// assert_eq!(top_docs.len(), 2);- /// assert_eq!(top_docs[0].1, DocAddress::new(0, 4));- /// assert_eq!(top_docs[1].1, DocAddress::new(0, 3));- /// Ok(())- /// # }- /// ```#[must_use]pub fn and_offset(self, offset: usize) -> TopDocs {TopDocs(self.0.and_offset(offset))@@ -249,64 +183,6 @@ impl TopDocs {/// will return returns a monotonic u64-representation (ie. the order is still correct) of/// the requested field type.///- /// # Example- ///- /// ```rust- /// # use tantivy::schema::{Schema, FAST, TEXT};- /// # use tantivy::{doc, Index, DocAddress, Order};- /// # use tantivy::query::{Query, QueryParser};- /// use tantivy::Searcher;- /// use tantivy::collector::TopDocs;- ///- /// # fn main() -> tantivy::Result<()> {- /// # let mut schema_builder = Schema::builder();- /// # let title = schema_builder.add_text_field("title", TEXT);- /// # let rating = schema_builder.add_u64_field("rating", FAST);- /// # let schema = schema_builder.build();- /// #- /// # let index = Index::create_in_ram(schema);- /// # let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;- /// # index_writer.add_document(doc!(title => "The Name of the Wind", rating => 92u64))?;- /// # index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64))?;- /// # index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64))?;- /// # index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64))?;- /// # index_writer.commit()?;- /// # let reader = index.reader()?;- /// # let query = QueryParser::for_index(&index, vec![title]).parse_query("diary")?;- /// # let top_docs = docs_sorted_by_rating(&reader.searcher(), &query)?;- /// # assert_eq!(top_docs,- /// # vec![(97u64, DocAddress::new(0u32, 1)),- /// # (80u64, DocAddress::new(0u32, 3))]);- /// # Ok(())- /// # }- /// /// Searches the document matching the given query, and- /// /// collects the top 10 documents, order by the u64-`field`- /// /// given in argument.- /// fn docs_sorted_by_rating(searcher: &Searcher,- /// query: &dyn Query)- /// -> tantivy::Result<Vec<(u64, DocAddress)>> {- ///- /// // This is where we build our topdocs collector- /// //- /// // Note the `rating_field` needs to be a FAST field here.- /// let top_books_by_rating = TopDocs- /// ::with_limit(10)- /// .order_by_fast_field("rating", Order::Desc);- ///- /// // ... and here are our documents. Note this is a simple vec.- /// // The `u64` in the pair is the value of our fast field for- /// // each documents.- /// //- /// // The vec is sorted decreasingly by `sort_by_field`, and has a- /// // length of 10, or less if not enough documents matched the- /// // query.- /// let resulting_docs: Vec<(u64, DocAddress)> =- /// searcher.search(query, &top_books_by_rating)?;- ///- /// Ok(resulting_docs)- /// }- /// ```- ////// # See also////// To comfortably work with `u64`s, `i64`s, `f64`s, or `date`s, please refer to@@ -435,76 +311,6 @@ impl TopDocs {/// In more serious application, this tweaking could involve running a/// learning-to-rank model over various features///- /// ```rust- /// # use tantivy::schema::{Schema, FAST, TEXT};- /// # use tantivy::{doc, Index, DocAddress, DocId, Score};- /// # use tantivy::query::QueryParser;- /// use tantivy::SegmentReader;- /// use tantivy::collector::TopDocs;- /// use tantivy::schema::Field;- ///- /// fn create_schema() -> Schema {- /// let mut schema_builder = Schema::builder();- /// schema_builder.add_text_field("product_name", TEXT);- /// schema_builder.add_u64_field("popularity", FAST);- /// schema_builder.build()- /// }- ///- /// fn create_index() -> tantivy::Result<Index> {- /// let schema = create_schema();- /// let index = Index::create_in_ram(schema);- /// let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;- /// let product_name = index.schema().get_field("product_name").unwrap();- /// let popularity: Field = index.schema().get_field("popularity").unwrap();- /// index_writer.add_document(doc!(product_name => "The Diary of Muadib", popularity => 1u64))?;- /// index_writer.add_document(doc!(product_name => "A Dairy Cow", popularity => 10u64))?;- /// index_writer.add_document(doc!(product_name => "The Diary of a Young Girl", popularity => 15u64))?;- /// index_writer.commit()?;- /// Ok(index)- /// }- ///- /// let index = create_index().unwrap();- /// let product_name = index.schema().get_field("product_name").unwrap();- /// let popularity: Field = index.schema().get_field("popularity").unwrap();- ///- /// let user_query_str = "diary";- /// let query_parser = QueryParser::for_index(&index, vec![product_name]);- /// let query = query_parser.parse_query(user_query_str).unwrap();- ///- /// // This is where we build our collector with our custom score.- /// let top_docs_by_custom_score = TopDocs- /// ::with_limit(10)- /// .tweak_score(move |segment_reader: &SegmentReader| {- /// // The argument is a function that returns our scoring- /// // function.- /// //- /// // The point of this "mother" function is to gather all- /// // of the segment level information we need for scoring.- /// // Typically, fast_fields.- /// //- /// // In our case, we will get a reader for the popularity- /// // fast field. For simplicity we read the first or default value in the fast- /// // field.- /// let popularity_reader =- /// segment_reader.fast_fields().u64("popularity").unwrap().first_or_default_col(0);- ///- /// // We can now define our actual scoring function- /// move |doc: DocId, original_score: Score| {- /// let popularity: u64 = popularity_reader.get_val(doc);- /// // Well.. For the sake of the example we use a simple logarithm- /// // function.- /// let popularity_boost_score = ((2u64 + popularity) as Score).log2();- /// popularity_boost_score * original_score- /// }- /// });- /// let reader = index.reader().unwrap();- /// let searcher = reader.searcher();- /// // ... and here are our documents. Note this is a simple vec.- /// // The `Score` in the pair is our tweaked score.- /// let resulting_docs: Vec<(Score, DocAddress)> =- /// searcher.search(&query, &top_docs_by_custom_score).unwrap();- /// ```- ////// # See also/// - [custom_score(...)](TopDocs::custom_score)pub fn tweak_score<TScore, TScoreSegmentTweaker, TScoreTweaker>(@@ -539,85 +345,6 @@ impl TopDocs {/// It can be used if your search engine relies on a learning-to-rank model for instance,/// which does not rely on the term frequencies or positions as features.///- /// # Example- ///- /// ```rust- /// # use tantivy::schema::{Schema, FAST, TEXT};- /// # use tantivy::{doc, Index, DocAddress, DocId};- /// # use tantivy::query::QueryParser;- /// use tantivy::SegmentReader;- /// use tantivy::collector::TopDocs;- /// use tantivy::schema::Field;- ///- /// # fn create_schema() -> Schema {- /// # let mut schema_builder = Schema::builder();- /// # schema_builder.add_text_field("product_name", TEXT);- /// # schema_builder.add_u64_field("popularity", FAST);- /// # schema_builder.add_u64_field("boosted", FAST);- /// # schema_builder.build()- /// # }- /// #- /// # fn main() -> tantivy::Result<()> {- /// # let schema = create_schema();- /// # let index = Index::create_in_ram(schema);- /// # let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;- /// # let product_name = index.schema().get_field("product_name").unwrap();- /// #- /// let popularity: Field = index.schema().get_field("popularity").unwrap();- /// let boosted: Field = index.schema().get_field("boosted").unwrap();- /// # index_writer.add_document(doc!(boosted=>1u64, product_name => "The Diary of Muadib", popularity => 1u64))?;- /// # index_writer.add_document(doc!(boosted=>0u64, product_name => "A Dairy Cow", popularity => 10u64))?;- /// # index_writer.add_document(doc!(boosted=>0u64, product_name => "The Diary of a Young Girl", popularity => 15u64))?;- /// # index_writer.commit()?;- /// // ...- /// # let user_query = "diary";- /// # let query = QueryParser::for_index(&index, vec![product_name]).parse_query(user_query)?;- ///- /// // This is where we build our collector with our custom score.- /// let top_docs_by_custom_score = TopDocs- /// ::with_limit(10)- /// .custom_score(move |segment_reader: &SegmentReader| {- /// // The argument is a function that returns our scoring- /// // function.- /// //- /// // The point of this "mother" function is to gather all- /// // of the segment level information we need for scoring.- /// // Typically, fast_fields.- /// //- /// // In our case, we will get a reader for the popularity- /// // fast field and a boosted field.- /// //- /// // We want to get boosted items score, and when we get- /// // a tie, return the item with the highest popularity.- /// //- /// // Note that this is implemented by using a `(u64, u64)`- /// // as a score.- /// let popularity_reader =- /// segment_reader.fast_fields().u64("popularity").unwrap().first_or_default_col(0);- /// let boosted_reader =- /// segment_reader.fast_fields().u64("boosted").unwrap().first_or_default_col(0);- ///- /// // We can now define our actual scoring function- /// move |doc: DocId| {- /// let popularity: u64 = popularity_reader.get_val(doc);- /// let boosted: u64 = boosted_reader.get_val(doc);- /// // Score do not have to be `f64` in tantivy.- /// // Here we return a couple to get lexicographical order- /// // for free.- /// (boosted, popularity)- /// }- /// });- /// # let reader = index.reader()?;- /// # let searcher = reader.searcher();- /// // ... and here are our documents. Note this is a simple vec.- /// // The `Score` in the pair is our tweaked score.- /// let resulting_docs: Vec<((u64, u64), DocAddress)> =- /// searcher.search(&*query, &top_docs_by_custom_score)?;- ///- /// # Ok(())- /// # }- /// ```- ////// # See also/// - [tweak_score(...)](TopDocs::tweak_score)pub fn custom_score<TScore, TCustomSegmentScorer, TCustomScorer>(diff --git a/crates/tantivy/src/error.rs b/crates/tantivy/src/error.rsindex ade94d54..6eeca6ec 100644--- a/crates/tantivy/src/error.rs+++ b/crates/tantivy/src/error.rs@@ -10,8 +10,11 @@ use crate::directory::error::{Incompatibility, LockError, OpenDirectoryError, OpenReadError, OpenWriteError,};use crate::fastfield::FastFieldNotAvailableError;+use crate::schema;use crate::schema::document::DeserializeError;-use crate::{query, schema};++#[cfg(test)]+use crate::query;/// Represents a `DataCorruption` error.///@@ -129,6 +132,7 @@ impl From<LockError> for TantivyError {}}+#[cfg(test)]impl From<query::QueryParserError> for TantivyError {fn from(parsing_error: query::QueryParserError) -> TantivyError {TantivyError::InvalidArgument(format!("Query is invalid. {parsing_error:?}"))diff --git a/crates/tantivy/src/indexer/index_writer.rs b/crates/tantivy/src/indexer/index_writer.rsindex 5b96227b..a6c3924c 100644--- a/crates/tantivy/src/indexer/index_writer.rs+++ b/crates/tantivy/src/indexer/index_writer.rs@@ -467,37 +467,6 @@ impl<D: Document> IndexWriter<D> {/// Requires `commit`ing/// Enables users to rebuild the index,/// by clearing and resubmitting necessary documents- ///- /// ```rust- /// use tantivy::collector::TopDocs;- /// use tantivy::query::QueryParser;- /// use tantivy::schema::*;- /// use tantivy::{doc, Index};- ///- /// fn main() -> tantivy::Result<()> {- /// let mut schema_builder = Schema::builder();- /// let title = schema_builder.add_text_field("title", TEXT | STORED);- /// let schema = schema_builder.build();- ///- /// let index = Index::create_in_ram(schema.clone());- ///- /// let mut index_writer = index.writer_with_num_threads(1, 50_000_000)?;- /// index_writer.add_document(doc!(title => "The modern Promotheus"))?;- /// index_writer.commit()?;- ///- /// let clear_res = index_writer.delete_all_documents().unwrap();- /// // have to commit, otherwise deleted terms remain available- /// index_writer.commit()?;- ///- /// let searcher = index.reader()?.searcher();- /// let query_parser = QueryParser::for_index(&index, vec![title]);- /// let query_promo = query_parser.parse_query("Promotheus")?;- /// let top_docs_promo = searcher.search(&query_promo, &TopDocs::with_limit(1))?;- ///- /// assert!(top_docs_promo.is_empty());- /// Ok(())- /// }- /// ```pub fn delete_all_documents(&self) -> crate::Result<Opstamp> {// Delete segmentsself.segment_updater.remove_all_segments();diff --git a/crates/tantivy/src/lib.rs b/crates/tantivy/src/lib.rsindex 14e52c50..394d95f4 100644--- a/crates/tantivy/src/lib.rs+++ b/crates/tantivy/src/lib.rs@@ -13,90 +13,6 @@//! Tantivy is a search engine library.//! Think `Lucene`, but in Rust.//!-//! ```rust-//! # use std::path::Path;-//! # use tempfile::TempDir;-//! # use tantivy::collector::TopDocs;-//! # use tantivy::query::QueryParser;-//! # use tantivy::schema::*;-//! # use tantivy::{doc, DocAddress, Index, IndexWriter, Score};-//! #-//! # fn main() {-//! # // Let's create a temporary directory for the-//! # // sake of this example-//! # if let Ok(dir) = TempDir::new() {-//! # run_example(dir.path()).unwrap();-//! # dir.close().unwrap();-//! # }-//! # }-//! #-//! # fn run_example(index_path: &Path) -> tantivy::Result<()> {-//! // First we need to define a schema ...-//!-//! // `TEXT` means the field should be tokenized and indexed,-//! // along with its term frequency and term positions.-//! //-//! // `STORED` means that the field will also be saved-//! // in a compressed, row-oriented key-value store.-//! // This store is useful to reconstruct the-//! // documents that were selected during the search phase.-//! let mut schema_builder = Schema::builder();-//! let title = schema_builder.add_text_field("title", TEXT | STORED);-//! let body = schema_builder.add_text_field("body", TEXT);-//! let schema = schema_builder.build();-//!-//! // Indexing documents-//!-//! let index = Index::create_in_dir(index_path, schema.clone())?;-//!-//! // Here we use a buffer of 100MB that will be split-//! // between indexing threads.-//! let mut index_writer: IndexWriter = index.writer(100_000_000)?;-//!-//! // Let's index one documents!-//! index_writer.add_document(doc!(-//! title => "The Old Man and the Sea",-//! body => "He was an old man who fished alone in a skiff in \-//! the Gulf Stream and he had gone eighty-four days \-//! now without taking a fish."-//! ))?;-//!-//! // We need to call .commit() explicitly to force the-//! // index_writer to finish processing the documents in the queue,-//! // flush the current index to the disk, and advertise-//! // the existence of new documents.-//! index_writer.commit()?;-//!-//! // # Searching-//!-//! let reader = index.reader()?;-//!-//! let searcher = reader.searcher();-//!-//! let query_parser = QueryParser::for_index(&index, vec![title, body]);-//!-//! // QueryParser may fail if the query is not in the right-//! // format. For user facing applications, this can be a problem.-//! // A ticket has been opened regarding this problem.-//! let query = query_parser.parse_query("sea whale")?;-//!-//! // Perform search.-//! // `topdocs` contains the 10 most relevant doc ids, sorted by decreasing scores...-//! let top_docs: Vec<(Score, DocAddress)> =-//! searcher.search(&query, &TopDocs::with_limit(10))?;-//!-//! for (_score, doc_address) in top_docs {-//! // Retrieve the actual content of documents given its `doc_address`.-//! let retrieved_doc = searcher.doc::<TantivyDocument>(doc_address)?;-//! println!("{}", retrieved_doc.to_json(&schema));-//! }-//!-//! # Ok(())-//! # }-//! ```-//!-//!-//!//! A good place for you to get started is to check out//! the example code (//! [literate programming](https://tantivy-search.github.io/examples/basic_search.html) /diff --git a/crates/tantivy/src/query/mod.rs b/crates/tantivy/src/query/mod.rsindex abe84d85..4083627a 100644--- a/crates/tantivy/src/query/mod.rs+++ b/crates/tantivy/src/query/mod.rs@@ -16,7 +16,6 @@ mod more_like_this;mod phrase_prefix_query;mod phrase_query;mod query;-mod query_parser;mod range_query;mod reqopt_scorer;mod scorer;@@ -28,6 +27,9 @@ mod weight;#[cfg(test)]mod vec_docset;+#[cfg(test)]+mod query_parser;+pub mod grammar;pub(crate) mod score_combiner;pub use grammar::Occur;@@ -52,6 +54,7 @@ pub use self::more_like_this::{MoreLikeThisQuery, MoreLikeThisQueryBuilder};pub use self::phrase_prefix_query::PhrasePrefixQuery;pub use self::phrase_query::PhraseQuery;pub use self::query::{EnableScoring, Query, QueryClone};+#[cfg(test)]pub use self::query_parser::{QueryParser, QueryParserError};pub use self::range_query::{FastFieldRangeWeight, IPFastFieldRangeWeight, RangeQuery};pub use self::reqopt_scorer::RequiredOptionalScorer;diff --git a/crates/tantivy/src/snippet/mod.rs b/crates/tantivy/src/snippet/mod.rsindex 16edac04..16d54953 100644--- a/crates/tantivy/src/snippet/mod.rs+++ b/crates/tantivy/src/snippet/mod.rs@@ -8,46 +8,6 @@//! It scans a document and constructs a snippet, which consists of sections where the search terms//! have been found, stitched together with "..." in between sections if necessary.//!-//! ## Example-//!-//! ```rust-//! # use tantivy::query::QueryParser;-//! # use tantivy::schema::{Schema, TEXT};-//! # use tantivy::{doc, Index};-//! use tantivy::snippet::SnippetGenerator;-//!-//! # fn main() -> tantivy::Result<()> {-//! # let mut schema_builder = Schema::builder();-//! # let text_field = schema_builder.add_text_field("text", TEXT);-//! # let schema = schema_builder.build();-//! # let index = Index::create_in_ram(schema);-//! # let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;-//! # let doc = doc!(text_field => r#"Comme je descendais des Fleuves impassibles,-//! # Je ne me sentis plus guidé par les haleurs :-//! # Des Peaux-Rouges criards les avaient pris pour cibles,-//! # Les ayant cloués nus aux poteaux de couleurs.-//! #-//! # J'étais insoucieux de tous les équipages,-//! # Porteur de blés flamands ou de cotons anglais.-//! # Quand avec mes haleurs ont fini ces tapages,-//! # Les Fleuves m'ont laissé descendre où je voulais.-//! # "#);-//! # index_writer.add_document(doc.clone())?;-//! # index_writer.commit()?;-//! # let query_parser = QueryParser::for_index(&index, vec![text_field]);-//! // ...-//! let query = query_parser.parse_query("haleurs flamands").unwrap();-//! # let reader = index.reader()?;-//! # let searcher = reader.searcher();-//! let mut snippet_generator = SnippetGenerator::create(&searcher, &*query, text_field)?;-//! snippet_generator.set_max_num_chars(100);-//! let snippet = snippet_generator.snippet_from_doc(&doc);-//! let snippet_html: String = snippet.to_html();-//! assert_eq!(snippet_html, "Comme je descendais des Fleuves impassibles,\n Je ne me sentis plus guidé par les <b>haleurs</b> :\n Des");-//! # Ok(())-//! # }-//! ```-//!//! You can also specify the maximum number of characters for the snippets generated with the//! `set_max_num_chars` method. By default, this limit is set to 150.//!@@ -304,47 +264,6 @@ fn is_sorted(mut it: impl Iterator<Item = usize>) -> bool {true}-/// `SnippetGenerator`-///-/// # Example-///-/// ```rust-/// # use tantivy::query::QueryParser;-/// # use tantivy::schema::{Schema, TEXT};-/// # use tantivy::{doc, Index};-/// use tantivy::snippet::SnippetGenerator;-///-/// # fn main() -> tantivy::Result<()> {-/// # let mut schema_builder = Schema::builder();-/// # let text_field = schema_builder.add_text_field("text", TEXT);-/// # let schema = schema_builder.build();-/// # let index = Index::create_in_ram(schema);-/// # let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;-/// # let doc = doc!(text_field => r#"Comme je descendais des Fleuves impassibles,-/// # Je ne me sentis plus guidé par les haleurs :-/// # Des Peaux-Rouges criards les avaient pris pour cibles,-/// # Les ayant cloués nus aux poteaux de couleurs.-/// #-/// # J'étais insoucieux de tous les équipages,-/// # Porteur de blés flamands ou de cotons anglais.-/// # Quand avec mes haleurs ont fini ces tapages,-/// # Les Fleuves m'ont laissé descendre où je voulais.-/// # "#);-/// # index_writer.add_document(doc.clone())?;-/// # index_writer.commit()?;-/// # let query_parser = QueryParser::for_index(&index, vec![text_field]);-/// // ...-/// let query = query_parser.parse_query("haleurs flamands").unwrap();-/// # let reader = index.reader()?;-/// # let searcher = reader.searcher();-/// let mut snippet_generator = SnippetGenerator::create(&searcher, &*query, text_field)?;-/// snippet_generator.set_max_num_chars(100);-/// let snippet = snippet_generator.snippet_from_doc(&doc);-/// let snippet_html: String = snippet.to_html();-/// assert_eq!(snippet_html, "Comme je descendais des Fleuves impassibles,\n Je ne me sentis plus guidé par les <b>haleurs</b> :\n Des");-/// # Ok(())-/// # }-/// ```pub struct SnippetGenerator {terms_text: BTreeMap<String, Score>,tokenizer: TextAnalyzer,