proper config base

This commit is contained in:
Milim 2024-08-15 07:53:26 +02:00
parent c9fe79eb12
commit 99a1089df9
No known key found for this signature in database
10 changed files with 114 additions and 203 deletions

262
Cargo.lock generated
View file

@ -278,18 +278,6 @@ dependencies = [
"alloc-no-stdlib",
]
[[package]]
name = "anes"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstyle"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
[[package]]
name = "anyhow"
version = "1.0.86"
@ -327,6 +315,15 @@ dependencies = [
"syn 2.0.74",
]
[[package]]
name = "atomic"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994"
dependencies = [
"bytemuck",
]
[[package]]
name = "atomic-waker"
version = "1.1.2"
@ -447,6 +444,12 @@ version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
[[package]]
name = "bytemuck"
version = "1.16.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "102087e286b4677862ea56cf8fc58bb2cdfa8725c40ffb80fe3a008eb7f2fc83"
[[package]]
name = "byteorder"
version = "1.5.0"
@ -510,12 +513,6 @@ dependencies = [
"serde_json",
]
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
version = "1.1.11"
@ -537,58 +534,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "ciborium"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
dependencies = [
"ciborium-io",
"ciborium-ll",
"serde",
]
[[package]]
name = "ciborium-io"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
[[package]]
name = "ciborium-ll"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
dependencies = [
"ciborium-io",
"half",
]
[[package]]
name = "clap"
version = "4.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11d8838454fda655dafd3accb2b6e2bea645b9e4078abe84a22ceb947235c5cc"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
version = "4.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6"
dependencies = [
"anstyle",
"clap_lex",
]
[[package]]
name = "clap_lex"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
[[package]]
name = "cloudabi"
version = "0.0.3"
@ -677,40 +622,6 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "criterion"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
dependencies = [
"anes",
"cast",
"ciborium",
"clap",
"criterion-plot",
"is-terminal",
"itertools",
"num-traits",
"once_cell",
"oorandom",
"regex",
"serde",
"serde_derive",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
dependencies = [
"cast",
"itertools",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.13"
@ -774,12 +685,6 @@ version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]]
name = "crunchy"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
[[package]]
name = "crypto-common"
version = "0.1.6"
@ -993,6 +898,19 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
[[package]]
name = "figment"
version = "0.10.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3"
dependencies = [
"atomic",
"pear",
"serde",
"uncased",
"version_check",
]
[[package]]
name = "flate2"
version = "1.0.31"
@ -1222,16 +1140,6 @@ dependencies = [
"tracing",
]
[[package]]
name = "half"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888"
dependencies = [
"cfg-if 1.0.0",
"crunchy",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
@ -1250,12 +1158,6 @@ version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
[[package]]
name = "hermit-abi"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc"
[[package]]
name = "html5ever"
version = "0.23.0"
@ -1529,6 +1431,12 @@ dependencies = [
"hashbrown 0.14.5",
]
[[package]]
name = "inlinable_string"
version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb"
[[package]]
name = "io-uring"
version = "0.6.4"
@ -1554,26 +1462,6 @@ version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
[[package]]
name = "is-terminal"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b"
dependencies = [
"hermit-abi 0.4.0",
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "itertools"
version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "0.4.8"
@ -1854,7 +1742,7 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
dependencies = [
"hermit-abi 0.3.9",
"hermit-abi",
"libc",
"log",
"wasi 0.11.0+wasi-snapshot-preview1",
@ -1913,22 +1801,13 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]]
name = "num-traits"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg 1.3.0",
]
[[package]]
name = "num_cpus"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
"hermit-abi 0.3.9",
"hermit-abi",
"libc",
]
@ -1947,12 +1826,6 @@ version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "oorandom"
version = "11.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"
[[package]]
name = "openssl"
version = "0.10.66"
@ -2052,6 +1925,29 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "pear"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467"
dependencies = [
"inlinable_string",
"pear_codegen",
"yansi",
]
[[package]]
name = "pear_codegen"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147"
dependencies = [
"proc-macro2 1.0.86",
"proc-macro2-diagnostics",
"quote 1.0.36",
"syn 2.0.74",
]
[[package]]
name = "percent-encoding"
version = "1.0.1"
@ -2282,6 +2178,19 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "proc-macro2-diagnostics"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
dependencies = [
"proc-macro2 1.0.86",
"quote 1.0.36",
"syn 2.0.74",
"version_check",
"yansi",
]
[[package]]
name = "publicsuffix"
version = "1.5.6"
@ -3292,16 +3201,6 @@ dependencies = [
"time-core",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "tinyvec"
version = "1.8.0"
@ -3591,6 +3490,15 @@ version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
[[package]]
name = "uncased"
version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697"
dependencies = [
"version_check",
]
[[package]]
name = "unicase"
version = "2.7.0"
@ -3839,10 +3747,10 @@ dependencies = [
"async-once-cell",
"async-trait",
"cfg-if 1.0.0",
"criterion",
"env_logger",
"error-stack",
"fake-useragent",
"figment",
"futures 0.3.30",
"keyword_extraction",
"log",
@ -4080,6 +3988,12 @@ dependencies = [
"winapi-build",
]
[[package]]
name = "yansi"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
[[package]]
name = "zerocopy"
version = "0.7.35"

View file

@ -56,17 +56,14 @@ async-compression = { version = "0.4.11", default-features = false, features = [
cfg-if = { version = "1.0.0", default-features = false, optional = true }
keyword_extraction = { version = "1.4.3", default-features = false, features = [
"tf_idf",
] }
figment = { version = "0.10", features = ["env"] }
stop-words = { version = "0.8.0", default-features = false, features = ["iso"] }
thesaurus = { version = "0.5.2", default-features = false, optional = true, features = [
"moby",
] }
[dev-dependencies]
criterion = { version = "0.5.1", default-features = false }
tempfile = { version = "3.10.1", default-features = false }
[features]

View file

@ -1,25 +1,23 @@
//! This module provides the functionality to cache the aggregated results fetched and aggregated
//! from the upstream search engines in a json format.
use error_stack::Report;
use mini_moka::sync::Cache as MokaCache;
use mini_moka::sync::ConcurrentCacheExt;
use std::time::Duration;
use crate::{config::parser::Config, models::aggregation_models::SearchResults};
use crate::{config::Config, models::aggregation_models::SearchResults};
impl Into<SearchResults> for Vec<u8> {
fn into(self) -> SearchResults {
serde_json::from_slice(&self)
impl From<Vec<u8>> for SearchResults {
fn from(v: Vec<u8>) -> SearchResults {
serde_json::from_slice(&v)
.expect("well, this can only be caused by memory corruption so good luck")
}
}
impl Into<Vec<u8>> for &SearchResults {
fn into(self) -> Vec<u8> {
serde_json::to_vec(self).expect("somehow failed to serialize search results")
impl From<&SearchResults> for Vec<u8> {
fn from(v: &SearchResults) -> Vec<u8> {
serde_json::to_vec(v).expect("somehow failed to serialize search results")
}
}

View file

@ -1,11 +1,12 @@
//! This module provides the functionality to parse the lua config and convert the config options
//! into rust readable form.
//! Config module
use figment::{providers::Serialized, Figment};
use serde::{Deserialize, Serialize};
use crate::models::parser_models::Style;
use std::collections::HashMap;
/// A named struct which stores the parsed config file options.
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct Config {
/// It stores the parsed port number option on which the server should launch.
pub port: u16,
@ -58,6 +59,9 @@ impl Default for Config {
impl Config {
/// Creates a new config based on the environment variables.
pub fn parse() -> Self {
Self::default()
Figment::from(Serialized::defaults(Config::default()))
.merge(figment::providers::Env::prefixed("SEARCH"))
.extract()
.unwrap()
}
}

View file

@ -1,4 +0,0 @@
//! This module provides the modules which handles the functionality to parse the lua config
//! and convert the config options into rust readable form.
pub mod parser;

View file

@ -26,7 +26,7 @@ use actix_web::{
middleware::{Compress, Logger},
web, App, HttpServer,
};
use config::parser::Config;
use config::Config;
use handler::{file_path, FileType};
/// Runs the web server

View file

@ -1,6 +1,8 @@
//! This module provides public models for handling, storing and serializing parsed config file
//! options from config.lua by grouping them together.
use serde::{Deserialize, Serialize};
/// A named struct which stores,deserializes, serializes and groups the parsed config file options
/// of theme and colorscheme names into the Style struct which derives the `Clone`, `Serialize`
/// and Deserialize traits where the `Clone` trait is derived for allowing the struct to be
@ -10,7 +12,7 @@
/// order to allow the deserializing the json back to struct in aggregate function in
/// aggregator.rs and create a new struct out of it and then serialize it back to json and pass
/// it to the template files.
#[derive(Default, Debug, Clone)]
#[derive(Default, Debug, Clone, Deserialize, Serialize)]
pub struct Style {
/// It stores the parsed theme option used to set a theme for the website.
pub theme: String,

View file

@ -2,7 +2,7 @@
//! search engines and then removes duplicate results.
use super::user_agent::random_user_agent;
use crate::config::parser::Config;
use crate::config::Config;
use crate::models::{
aggregation_models::{EngineErrorInfo, SearchResult, SearchResults},
engine_models::{EngineError, EngineHandler},
@ -69,7 +69,7 @@ type FutureVec =
pub async fn aggregate(
query: &str,
page: u32,
config: actix_web::web::Data<crate::config::parser::Config>,
config: actix_web::web::Data<Config>,
upstream_search_engines: &[EngineHandler],
) -> Result<SearchResults, Box<dyn std::error::Error>> {
let client = CLIENT.get_or_init(|| {

View file

@ -3,7 +3,7 @@
//! when requested.
use crate::{
config::parser::Config,
config::Config,
handler::{file_path, FileType},
};
use actix_web::{get, http::header::ContentType, web, HttpRequest, HttpResponse};

View file

@ -2,7 +2,7 @@
use crate::{
cache::Cache,
config::parser::Config,
config::Config,
models::{
aggregation_models::SearchResults,
engine_models::EngineHandler,
@ -144,7 +144,7 @@ pub async fn search(
/// It returns the `SearchResults` struct if the search results could be successfully fetched from
/// the cache or from the upstream search engines otherwise it returns an appropriate error.
async fn results(
config: web::Data<crate::config::parser::Config>,
config: web::Data<Config>,
cache: web::Data<crate::cache::Cache>,
query: &str,
page: u32,