web search done right
Added license to cargo.toml
Mikkel Denker0d97184026d2
diff --git a/Cargo.toml b/Cargo.tomlindex 48afa994..f39cd7e7 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -2,6 +2,8 @@name = "cuely"version = "0.1.0"edition = "2021"+license = "AGPL-3.0-or-later"+authors = ['Cuely ApS. <[email protected]>']# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.htmldiff --git a/src/bin/cuely.rs b/src/bin/cuely.rsindex a0812962..59b932f7 100644--- a/src/bin/cuely.rs+++ b/src/bin/cuely.rs@@ -1,20 +1,18 @@-/*- * Cuely is an open source web search engine.- * Copyright (C) 2022 Cuely ApS- *- * This program is free software: you can redistribute it and/or modify- * it under the terms of the GNU Affero General Public License as- * published by the Free Software Foundation, either version 3 of the- * License, or (at your option) any later version.- *- * This program is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the- * GNU Affero General Public License for more details.- *- * You should have received a copy of the GNU Affero General Public License- * along with this program. If not, see <https://www.gnu.org/licenses/>.- */+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.use cuely::{Config, Indexer, Mode};use std::fs;diff --git a/src/indexer.rs b/src/indexer.rsindex 14cf0a2f..b0bacbc8 100644--- a/src/indexer.rs+++ b/src/indexer.rs@@ -1,20 +1,18 @@-/*- * Cuely is an open source web search engine.- * Copyright (C) 2022 Cuely ApS- *- * This program is free software: you can redistribute it and/or modify- * it under the terms of the GNU Affero General Public License as- * published by the Free Software Foundation, either version 3 of the- * License, or (at your option) any later version.- *- * This program is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the- * GNU Affero General Public License for more details.- *- * You should have received a copy of the GNU Affero General Public License- * along with this program. If not, see <https://www.gnu.org/licenses/>.- */+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.use futures::prelude::*;use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};use rusoto_core::Region;diff --git a/src/lib.rs b/src/lib.rsindex 7588386f..bbc864b8 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -1,20 +1,18 @@-/*- * Cuely is an open source web search engine.- * Copyright (C) 2022 Cuely ApS- *- * This program is free software: you can redistribute it and/or modify- * it under the terms of the GNU Affero General Public License as- * published by the Free Software Foundation, either version 3 of the- * License, or (at your option) any later version.- *- * This program is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the- * GNU Affero General Public License for more details.- *- * You should have received a copy of the GNU Affero General Public License- * along with this program. If not, see <https://www.gnu.org/licenses/>.- */+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.use serde::Deserialize;use std::io;use std::num::ParseIntError;diff --git a/src/query.rs b/src/query.rsindex 4be3a5b2..e8e7e0a4 100644--- a/src/query.rs+++ b/src/query.rs@@ -1,20 +1,18 @@-/*- * Cuely is an open source web search engine.- * Copyright (C) 2022 Cuely ApS- *- * This program is free software: you can redistribute it and/or modify- * it under the terms of the GNU Affero General Public License as- * published by the Free Software Foundation, either version 3 of the- * License, or (at your option) any later version.- *- * This program is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the- * GNU Affero General Public License for more details.- *- * You should have received a copy of the GNU Affero General Public License- * along with this program. If not, see <https://www.gnu.org/licenses/>.- */+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{Error, Result};use nom::{bytes::complete::{tag, take_while},diff --git a/src/ranking.rs b/src/ranking.rsindex 2d26c8f7..1325e41c 100644--- a/src/ranking.rs+++ b/src/ranking.rs@@ -1,20 +1,19 @@-/*- * Cuely is an open source web search engine.- * Copyright (C) 2022 Cuely ApS- *- * This program is free software: you can redistribute it and/or modify- * it under the terms of the GNU Affero General Public License as- * published by the Free Software Foundation, either version 3 of the- * License, or (at your option) any later version.- *- * This program is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the- * GNU Affero General Public License for more details.- *- * You should have received a copy of the GNU Affero General Public License- * along with this program. If not, see <https://www.gnu.org/licenses/>.- */+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use crate::schema::Field;use tantivy::fastfield::FastFieldReader;use tantivy::{diff --git a/src/schema.rs b/src/schema.rsindex f13f2cbe..5807028d 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -1,20 +1,19 @@-/*- * Cuely is an open source web search engine.- * Copyright (C) 2022 Cuely ApS- *- * This program is free software: you can redistribute it and/or modify- * it under the terms of the GNU Affero General Public License as- * published by the Free Software Foundation, either version 3 of the- * License, or (at your option) any later version.- *- * This program is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the- * GNU Affero General Public License for more details.- *- * You should have received a copy of the GNU Affero General Public License- * along with this program. If not, see <https://www.gnu.org/licenses/>.- */+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use tantivy::schema::{Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,};diff --git a/src/search_index.rs b/src/search_index.rsindex a27b6721..59e5a83b 100644--- a/src/search_index.rs+++ b/src/search_index.rs@@ -1,20 +1,18 @@-/*- * Cuely is an open source web search engine.- * Copyright (C) 2022 Cuely ApS- *- * This program is free software: you can redistribute it and/or modify- * it under the terms of the GNU Affero General Public License as- * published by the Free Software Foundation, either version 3 of the- * License, or (at your option) any later version.- *- * This program is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the- * GNU Affero General Public License for more details.- *- * You should have received a copy of the GNU Affero General Public License- * along with this program. If not, see <https://www.gnu.org/licenses/>.- */+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.use tantivy::collector::Count;use tantivy::{DocAddress, Document, LeasedItem, Searcher};diff --git a/src/warc.rs b/src/warc.rsindex 37b62a6f..548191c8 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -1,20 +1,18 @@-/*- * Cuely is an open source web search engine.- * Copyright (C) 2022 Cuely ApS- *- * This program is free software: you can redistribute it and/or modify- * it under the terms of the GNU Affero General Public License as- * published by the Free Software Foundation, either version 3 of the- * License, or (at your option) any later version.- *- * This program is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the- * GNU Affero General Public License for more details.- *- * You should have received a copy of the GNU Affero General Public License- * along with this program. If not, see <https://www.gnu.org/licenses/>.- */+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{Error, Result};use std::collections::BTreeMap;use std::io::{BufRead, BufReader, Read};diff --git a/src/webpage.rs b/src/webpage.rsindex 2a515ba8..d9c00444 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -1,20 +1,18 @@-/*- * Cuely is an open source web search engine.- * Copyright (C) 2022 Cuely ApS- *- * This program is free software: you can redistribute it and/or modify- * it under the terms of the GNU Affero General Public License as- * published by the Free Software Foundation, either version 3 of the- * License, or (at your option) any later version.- *- * This program is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the- * GNU Affero General Public License for more details.- *- * You should have received a copy of the GNU Affero General Public License- * along with this program. If not, see <https://www.gnu.org/licenses/>.- */+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{Error, Result};use itertools::Itertools;use scraper::Html as ScraperHtml;