feat: initial commit.

.gitignore

@@ -0,0 +1,2 @@
+index/
+target/

Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "search"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = "1.0.104"
+clap = { version = "4.6.6", features = ["derive"] }
+serde = { version = "1.0.229", features = ["derive"] }
+serde_yaml = "0.9.34"
+tantivy = "0.26.1"

README.md

@@ -0,0 +1,24 @@
+# Search with tantivy
+
+This demo takes the base_data.yml, indexes it and provides
+a search in this data.
+
+## Index
+
+Create an index directory and then execute the indexing:
+
+```bash
+mkdir index
+cargo run -- index
+```
+
+## Search
+
+Afterwards you can search in the index.
+It will return a score and the result as JSON.
+
+```bash
+cargo run -- search "jazz music"
+[3.699133] {"description":["Jazzy music with some rainy atmosphere"],"id":["0004"],"title":["I am currently listening to Jazz"]}
+
+```

base_data.yml

@@ -0,0 +1,30 @@
+- id: "0001"
+  title: "Here are some words"
+  description: "An entry with some words"
+- id: "0002"
+  title: "Something about birds"
+  description: "I have seen some ducks yesterday"
+- id: "0003"
+  title: "My curtains are gray"
+  description: "Those curtains are blocking light"
+- id: "0004"
+  title: "I am currently listening to Jazz"
+  description: "Jazzy music with some rainy atmosphere"
+- id: "0005"
+  title: "Drinking some sugarless products"
+  description: "Naming it would be marketing"
+- id: "0006"
+  title: "Connected some lan cables today"
+  description: "Working with CAT 6"
+- id: "0007"
+  title: "I should have used AI to generate this"
+  description: "On the other side, this is some creative work"
+- id: "0008"
+  title: "But does this work make sense?"
+  description: "What in life does make sense?"
+- id: "0009"
+  title: "Philosophy"
+  description: "Something about Epictetus"
+- id: "0010"
+  title: "The last sentence"
+  description: "On how to dump data out of your head"

src/main.rs

@@ -0,0 +1,119 @@
+use anyhow::{Context, Result};
+use clap::Parser;
+use serde::Deserialize;
+use tantivy::{
+    DocAddress, Document, Index, IndexReader, IndexWriter, Score, TantivyDocument,
+    collector::TopDocs,
+    doc,
+    query::QueryParser,
+    schema::{Field, STORED, STRING, Schema, TEXT},
+};
+
+const BASE_DATA: &str = include_str!("../base_data.yml");
+
+#[derive(Parser)]
+enum Args {
+    #[command(about = "Build an index")]
+    Index,
+    #[command(about = "Execute the search")]
+    Search {
+        #[arg(help = "A search query")]
+        search: String,
+    },
+}
+
+#[derive(Deserialize)]
+struct BaseDataEntry {
+    id: String,
+    title: String,
+    description: String,
+}
+
+struct IndexData {
+    index: Index,
+    schema: Schema,
+    id: Field,
+    title: Field,
+    description: Field,
+}
+
+fn main() -> Result<()> {
+    let args = Args::parse();
+
+    match args {
+        Args::Index => execute_index(),
+        Args::Search { search } => execute_search(search),
+    }
+}
+
+fn build_index_data(create: bool) -> Result<IndexData> {
+    let mut schema_builder = Schema::builder();
+
+    let id = schema_builder.add_text_field("id", STRING | STORED);
+    let title = schema_builder.add_text_field("title", TEXT | STORED);
+    let description = schema_builder.add_text_field("description", TEXT | STORED);
+
+    let schema = schema_builder.build();
+    let index = match create {
+        true => {
+            Index::create_in_dir("index", schema.clone()).context("Creating index in directory")?
+        }
+        false => Index::open_in_dir("index").context("Opening index in directory")?,
+    };
+
+    Ok(IndexData {
+        index,
+        schema,
+        id,
+        title,
+        description,
+    })
+}
+
+fn execute_index() -> Result<()> {
+    let index_data = build_index_data(true)?;
+    let index = index_data.index;
+    let id = index_data.id;
+    let title = index_data.title;
+    let description = index_data.description;
+
+    let mut index_writer: IndexWriter = index
+        .writer(100_000_000)
+        .context("Create an index writer")?;
+
+    let base_data: Vec<BaseDataEntry> = serde_yaml::from_str(BASE_DATA)?;
+    for data in base_data {
+        index_writer.add_document(doc!(
+            id => data.id,
+            title => data.title,
+            description => data.description,
+        ))?;
+    }
+
+    index_writer.commit()?;
+    Ok(())
+}
+
+fn execute_search(search: String) -> Result<()> {
+    let index_data = build_index_data(false)?;
+    let index = index_data.index;
+    let schema = index_data.schema;
+    let id = index_data.id;
+    let title = index_data.title;
+    let description = index_data.description;
+
+    let index_reader: IndexReader = index.reader().context("Create an index reader")?;
+    let searcher = index_reader.searcher();
+    let query_parser = QueryParser::for_index(&index, vec![id, title, description]);
+
+    let query = query_parser.parse_query(&search)?;
+    let top_docs: Vec<(Score, DocAddress)> =
+        searcher.search(&query, &TopDocs::with_limit(10).order_by_score())?;
+
+    for (score, doc_address) in top_docs {
+        let retrieved_doc = searcher.doc::<TantivyDocument>(doc_address)?;
+        println!("[{}] {}", score, retrieved_doc.to_json(&schema));
+    }
+
+    Ok(())
+}