From 4afc0d16f072bb84173f6845f3c5d78b316082c8 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 20:46:19 +0300 Subject: [PATCH 01/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20add?= =?UTF-8?q?=20dhat=20profiler=20configuration=20with=20feat=20flag=20(#180?= =?UTF-8?q?)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/websurfx.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index 75d0b8d..1f26af4 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -6,6 +6,11 @@ use std::net::TcpListener; use websurfx::{config::parser::Config, run}; +/// A dhat heap memory profiler +#[cfg(feature = "dhat-heap")] +#[global_allocator] +static ALLOC: dhat::Alloc = dhat::Alloc; + /// The function that launches the main server and registers all the routes of the website. /// /// # Error @@ -14,6 +19,10 @@ use websurfx::{config::parser::Config, run}; /// available for being used for other applications. #[actix_web::main] async fn main() -> std::io::Result<()> { + // A dhat heap profiler initialization. + #[cfg(feature = "dhat-heap")] + let _profiler = dhat::Profiler::new_heap(); + // Initialize the parsed config file. let config = Config::parse(false).unwrap(); From db93c316034d4dab2caab26a4c5a0adbe122a2c0 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 20:50:42 +0300 Subject: [PATCH 02/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20imple?= =?UTF-8?q?ment=20async=20pooling=20for=20redis=20connections=20(#180)(#17?= =?UTF-8?q?8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cache/cacher.rs | 126 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 103 insertions(+), 23 deletions(-) diff --git a/src/cache/cacher.rs b/src/cache/cacher.rs index 44d0710..6932dea 100644 --- a/src/cache/cacher.rs +++ b/src/cache/cacher.rs @@ -1,17 +1,26 @@ //! This module provides the functionality to cache the aggregated results fetched and aggregated //! from the upstream search engines in a json format. +use error_stack::Report; +use futures::future::try_join_all; use md5::compute; -use redis::{Client, Commands, Connection}; +use redis::{aio::ConnectionManager, AsyncCommands, Client, RedisError}; + +use super::error::PoolError; /// A named struct which stores the redis Connection url address to which the client will /// connect to. /// /// # Fields /// -/// * `redis_connection_url` - It stores the redis Connection url address. +/// * `connection_pool` - It stores a pool of connections ready to be used. +/// * `pool_size` - It stores the size of the connection pool (in other words the number of +/// connections that should be stored in the pool). +/// * `current_connection` - It stores the index of which connection is being used at the moment. pub struct RedisCache { - connection: Connection, + connection_pool: Vec, + pool_size: u8, + current_connection: u8, } impl RedisCache { @@ -19,11 +28,25 @@ impl RedisCache { /// /// # Arguments /// - /// * `redis_connection_url` - It stores the redis Connection url address. - pub fn new(redis_connection_url: String) -> Result> { + /// * `redis_connection_url` - It takes the redis Connection url address. + /// * `pool_size` - It takes the size of the connection pool (in other words the number of + /// connections that should be stored in the pool). + pub async fn new( + redis_connection_url: &str, + pool_size: u8, + ) -> Result> { let client = Client::open(redis_connection_url)?; - let connection = client.get_connection()?; - let redis_cache = RedisCache { connection }; + let mut tasks: Vec<_> = Vec::new(); + + for _ in 0..pool_size { + tasks.push(client.get_tokio_connection_manager()); + } + + let redis_cache = RedisCache { + connection_pool: try_join_all(tasks).await?, + pool_size, + current_connection: Default::default(), + }; Ok(redis_cache) } @@ -32,7 +55,7 @@ impl RedisCache { /// # Arguments /// /// * `url` - It takes an url as string. - fn hash_url(url: &str) -> String { + fn hash_url(&self, url: &str) -> String { format!("{:?}", compute(url)) } @@ -41,9 +64,42 @@ impl RedisCache { /// # Arguments /// /// * `url` - It takes an url as a string. - pub fn cached_json(&mut self, url: &str) -> Result> { - let hashed_url_string = Self::hash_url(url); - Ok(self.connection.get(hashed_url_string)?) + pub async fn cached_json(&mut self, url: &str) -> Result> { + self.current_connection = Default::default(); + let hashed_url_string: &str = &self.hash_url(url); + + let mut result: Result = self.connection_pool + [self.current_connection as usize] + .get(hashed_url_string) + .await; + + // Code to check whether the current connection being used is dropped with connection error + // or not. if it drops with the connection error then the current connection is replaced + // with a new connection from the pool which is then used to run the redis command then + // that connection is also checked whether it is dropped or not if it is not then the + // result is passed as a `Result` or else the same process repeats again and if all of the + // connections in the pool result in connection drop error then a custom pool error is + // returned. + loop { + match result { + Err(error) => match error.is_connection_dropped() { + true => { + self.current_connection += 1; + if self.current_connection == self.pool_size { + return Err(Report::new( + PoolError::PoolExhaustionWithConnectionDropError, + )); + } + result = self.connection_pool[self.current_connection as usize] + .get(hashed_url_string) + .await; + continue; + } + false => return Err(Report::new(PoolError::RedisError(error))), + }, + Ok(res) => return Ok(res), + } + } } /// A function which caches the results by using the hashed `url` as the key and @@ -54,21 +110,45 @@ impl RedisCache { /// /// * `json_results` - It takes the json results string as an argument. /// * `url` - It takes the url as a String. - pub fn cache_results( + pub async fn cache_results( &mut self, - json_results: String, + json_results: &str, url: &str, - ) -> Result<(), Box> { - let hashed_url_string = Self::hash_url(url); + ) -> Result<(), Report> { + self.current_connection = Default::default(); + let hashed_url_string: &str = &self.hash_url(url); - // put results_json into cache - self.connection.set(&hashed_url_string, json_results)?; + let mut result: Result<(), RedisError> = self.connection_pool + [self.current_connection as usize] + .set_ex(hashed_url_string, json_results, 60) + .await; - // Set the TTL for the key to 60 seconds - self.connection - .expire::(hashed_url_string, 60) - .unwrap(); - - Ok(()) + // Code to check whether the current connection being used is dropped with connection error + // or not. if it drops with the connection error then the current connection is replaced + // with a new connection from the pool which is then used to run the redis command then + // that connection is also checked whether it is dropped or not if it is not then the + // result is passed as a `Result` or else the same process repeats again and if all of the + // connections in the pool result in connection drop error then a custom pool error is + // returned. + loop { + match result { + Err(error) => match error.is_connection_dropped() { + true => { + self.current_connection += 1; + if self.current_connection == self.pool_size { + return Err(Report::new( + PoolError::PoolExhaustionWithConnectionDropError, + )); + } + result = self.connection_pool[self.current_connection as usize] + .set_ex(hashed_url_string, json_results, 60) + .await; + continue; + } + false => return Err(Report::new(PoolError::RedisError(error))), + }, + Ok(_) => return Ok(()), + } + } } } From 01d8c7ae4ccec15441162c071683396dc83fe769 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 20:52:16 +0300 Subject: [PATCH 03/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20add?= =?UTF-8?q?=20new=20pooling=20error=20type=20for=20pooling=20code=20(#180)?= =?UTF-8?q?(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cache/error.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/cache/error.rs diff --git a/src/cache/error.rs b/src/cache/error.rs new file mode 100644 index 0000000..efd87c9 --- /dev/null +++ b/src/cache/error.rs @@ -0,0 +1,40 @@ +//! This module provides the error enum to handle different errors associated while requesting data from +//! the redis server using an async connection pool. +use std::fmt; + +use redis::RedisError; + +/// A custom error type used for handling redis async pool associated errors. +/// +/// This enum provides variants three different categories of errors: +/// * `RedisError` - This variant handles all errors related to `RedisError`, +/// * `PoolExhaustionWithConnectionDropError` - This variant handles the error +/// which occurs when all the connections in the connection pool return a connection +/// dropped redis error. +#[derive(Debug)] +pub enum PoolError { + RedisError(RedisError), + PoolExhaustionWithConnectionDropError, +} + +impl fmt::Display for PoolError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + PoolError::RedisError(redis_error) => { + if let Some(detail) = redis_error.detail() { + write!(f, "{}", detail) + } else { + write!(f, "") + } + } + PoolError::PoolExhaustionWithConnectionDropError => { + write!( + f, + "Error all connections from the pool dropped with connection error" + ) + } + } + } +} + +impl error_stack::Context for PoolError {} From 5f1a43976f22f08c8ae24debcade6f6ad8c8c6b8 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 20:53:55 +0300 Subject: [PATCH 04/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20add?= =?UTF-8?q?=20error.rs=20module=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cache/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cache/mod.rs b/src/cache/mod.rs index de7dd4e..03c4155 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -1 +1,2 @@ pub mod cacher; +pub mod error; From e4476aae2865fa57f70d15f3efbd88c99cebeebb Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 20:55:34 +0300 Subject: [PATCH 05/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20repla?= =?UTF-8?q?ce=20rlua=20with=20mlua=20code=20implementation=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config/parser.rs | 84 ++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/src/config/parser.rs b/src/config/parser.rs index 4639013..fc0a861 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -5,7 +5,7 @@ use crate::handler::paths::{file_path, FileType}; use super::parser_models::Style; use log::LevelFilter; -use rlua::Lua; +use mlua::Lua; use std::{collections::HashMap, fs, thread::available_parallelism}; /// A named struct which stores the parsed config file options. @@ -63,53 +63,53 @@ impl Config { /// or io error if the config.lua file doesn't exists otherwise it returns a newly constructed /// Config struct with all the parsed config options from the parsed config file. pub fn parse(logging_initialized: bool) -> Result> { - Lua::new().context(|context| -> Result> { - let globals = context.globals(); + let lua = Lua::new(); + let globals = lua.globals(); - context - .load(&fs::read_to_string(file_path(FileType::Config)?)?) - .exec()?; + lua.load(&fs::read_to_string(file_path(FileType::Config)?)?) + .exec()?; - let parsed_threads: u8 = globals.get::<_, u8>("threads")?; + let parsed_threads: u8 = globals.get::<_, u8>("threads")?; - let debug: bool = globals.get::<_, bool>("debug")?; - let logging:bool= globals.get::<_, bool>("logging")?; + let debug: bool = globals.get::<_, bool>("debug")?; + let logging: bool = globals.get::<_, bool>("logging")?; - if !logging_initialized { - set_logging_level(debug, logging); - } + if !logging_initialized { + set_logging_level(debug, logging); + } - let threads: u8 = if parsed_threads == 0 { - let total_num_of_threads: usize = available_parallelism()?.get() / 2; - log::error!("Config Error: The value of `threads` option should be a non zero positive integer"); - log::error!("Falling back to using {} threads", total_num_of_threads); - total_num_of_threads as u8 - } else { - parsed_threads - }; + let threads: u8 = if parsed_threads == 0 { + let total_num_of_threads: usize = available_parallelism()?.get() / 2; + log::error!( + "Config Error: The value of `threads` option should be a non zero positive integer" + ); + log::error!("Falling back to using {} threads", total_num_of_threads); + total_num_of_threads as u8 + } else { + parsed_threads + }; - Ok(Config { - port: globals.get::<_, u16>("port")?, - binding_ip: globals.get::<_, String>("binding_ip")?, - style: Style::new( - globals.get::<_, String>("theme")?, - globals.get::<_, String>("colorscheme")?, - ), - redis_url: globals.get::<_, String>("redis_url")?, - aggregator: AggregatorConfig { - random_delay: globals.get::<_, bool>("production_use")?, - }, - logging, - debug, - upstream_search_engines: globals - .get::<_, HashMap>("upstream_search_engines")? - .into_iter() - .filter_map(|(key, value)| value.then_some(key)) - .filter_map(|engine| crate::engines::engine_models::EngineHandler::new(&engine)) - .collect(), - request_timeout: globals.get::<_, u8>("request_timeout")?, - threads, - }) + Ok(Config { + port: globals.get::<_, u16>("port")?, + binding_ip: globals.get::<_, String>("binding_ip")?, + style: Style::new( + globals.get::<_, String>("theme")?, + globals.get::<_, String>("colorscheme")?, + ), + redis_url: globals.get::<_, String>("redis_url")?, + aggregator: AggregatorConfig { + random_delay: globals.get::<_, bool>("production_use")?, + }, + logging, + debug, + upstream_search_engines: globals + .get::<_, HashMap>("upstream_search_engines")? + .into_iter() + .filter_map(|(key, value)| value.then_some(key)) + .filter_map(|engine| crate::engines::engine_models::EngineHandler::new(&engine)) + .collect(), + request_timeout: globals.get::<_, u8>("request_timeout")?, + threads, }) } } From 7b33744c9d1263b4c269925e0c1459ff452e68af Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 20:56:29 +0300 Subject: [PATCH 06/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20imple?= =?UTF-8?q?ment=20default=20trait=20for=20Style=20struct=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config/parser_models.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/parser_models.rs b/src/config/parser_models.rs index 0bc52d8..7528715 100644 --- a/src/config/parser_models.rs +++ b/src/config/parser_models.rs @@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize}; /// * `theme` - It stores the parsed theme option used to set a theme for the website. /// * `colorscheme` - It stores the parsed colorscheme option used to set a colorscheme for the /// theme being used. -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, Default)] pub struct Style { pub theme: String, pub colorscheme: String, From 4ccd0486e710864d6a35c11bc41e6fd6b31618dc Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 20:57:33 +0300 Subject: [PATCH 07/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20repla?= =?UTF-8?q?ce=20oncecell=20with=20oncelock=20from=20std=20library=20(#180)?= =?UTF-8?q?(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/handler/paths.rs | 112 ++++++++++++++++++++------------------ src/results/user_agent.rs | 38 +++++++------ 2 files changed, 79 insertions(+), 71 deletions(-) diff --git a/src/handler/paths.rs b/src/handler/paths.rs index 9b4fa07..91f7f94 100644 --- a/src/handler/paths.rs +++ b/src/handler/paths.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; use std::io::Error; use std::path::Path; +use std::sync::OnceLock; // ------- Constants -------- static PUBLIC_DIRECTORY_NAME: &str = "public"; @@ -20,57 +21,7 @@ pub enum FileType { Theme, } -static FILE_PATHS_FOR_DIFF_FILE_TYPES: once_cell::sync::Lazy>> = - once_cell::sync::Lazy::new(|| { - HashMap::from([ - ( - FileType::Config, - vec![ - format!( - "{}/.config/{}/{}", - std::env::var("HOME").unwrap(), - COMMON_DIRECTORY_NAME, - CONFIG_FILE_NAME - ), - format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME), - format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME), - ], - ), - ( - FileType::Theme, - vec![ - format!("/opt/websurfx/{}/", PUBLIC_DIRECTORY_NAME), - format!("./{}/", PUBLIC_DIRECTORY_NAME), - ], - ), - ( - FileType::AllowList, - vec![ - format!( - "{}/.config/{}/{}", - std::env::var("HOME").unwrap(), - COMMON_DIRECTORY_NAME, - ALLOWLIST_FILE_NAME - ), - format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, ALLOWLIST_FILE_NAME), - format!("./{}/{}", COMMON_DIRECTORY_NAME, ALLOWLIST_FILE_NAME), - ], - ), - ( - FileType::BlockList, - vec![ - format!( - "{}/.config/{}/{}", - std::env::var("HOME").unwrap(), - COMMON_DIRECTORY_NAME, - BLOCKLIST_FILE_NAME - ), - format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, BLOCKLIST_FILE_NAME), - format!("./{}/{}", COMMON_DIRECTORY_NAME, BLOCKLIST_FILE_NAME), - ], - ), - ]) - }); +static FILE_PATHS_FOR_DIFF_FILE_TYPES: OnceLock>> = OnceLock::new(); /// A helper function which returns an appropriate config file path checking if the config /// file exists on that path. @@ -95,11 +46,64 @@ static FILE_PATHS_FOR_DIFF_FILE_TYPES: once_cell::sync::Lazy Result { - let file_path = FILE_PATHS_FOR_DIFF_FILE_TYPES.get(&file_type).unwrap(); +pub fn file_path(file_type: FileType) -> Result<&'static str, Error> { + let file_path: &Vec = FILE_PATHS_FOR_DIFF_FILE_TYPES + .get_or_init(|| { + HashMap::from([ + ( + FileType::Config, + vec![ + format!( + "{}/.config/{}/{}", + std::env::var("HOME").unwrap(), + COMMON_DIRECTORY_NAME, + CONFIG_FILE_NAME + ), + format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME), + format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME), + ], + ), + ( + FileType::Theme, + vec![ + format!("/opt/websurfx/{}/", PUBLIC_DIRECTORY_NAME), + format!("./{}/", PUBLIC_DIRECTORY_NAME), + ], + ), + ( + FileType::AllowList, + vec![ + format!( + "{}/.config/{}/{}", + std::env::var("HOME").unwrap(), + COMMON_DIRECTORY_NAME, + ALLOWLIST_FILE_NAME + ), + format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, ALLOWLIST_FILE_NAME), + format!("./{}/{}", COMMON_DIRECTORY_NAME, ALLOWLIST_FILE_NAME), + ], + ), + ( + FileType::BlockList, + vec![ + format!( + "{}/.config/{}/{}", + std::env::var("HOME").unwrap(), + COMMON_DIRECTORY_NAME, + BLOCKLIST_FILE_NAME + ), + format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, BLOCKLIST_FILE_NAME), + format!("./{}/{}", COMMON_DIRECTORY_NAME, BLOCKLIST_FILE_NAME), + ], + ), + ]) + }) + .get(&file_type) + .unwrap(); + for (idx, _) in file_path.iter().enumerate() { if Path::new(file_path[idx].as_str()).exists() { - return Ok(file_path[idx].clone()); + return Ok(std::mem::take(&mut &*file_path[idx])); } } diff --git a/src/results/user_agent.rs b/src/results/user_agent.rs index 13166bf..3bfa05b 100644 --- a/src/results/user_agent.rs +++ b/src/results/user_agent.rs @@ -1,28 +1,32 @@ //! This module provides the functionality to generate random user agent string. +use std::sync::OnceLock; + use fake_useragent::{Browsers, UserAgents, UserAgentsBuilder}; -static USER_AGENTS: once_cell::sync::Lazy = once_cell::sync::Lazy::new(|| { - UserAgentsBuilder::new() - .cache(false) - .dir("/tmp") - .thread(1) - .set_browsers( - Browsers::new() - .set_chrome() - .set_safari() - .set_edge() - .set_firefox() - .set_mozilla(), - ) - .build() -}); +static USER_AGENTS: OnceLock = OnceLock::new(); /// A function to generate random user agent to improve privacy of the user. /// /// # Returns /// /// A randomly generated user agent string. -pub fn random_user_agent() -> String { - USER_AGENTS.random().to_string() +pub fn random_user_agent() -> &'static str { + USER_AGENTS + .get_or_init(|| { + UserAgentsBuilder::new() + .cache(false) + .dir("/tmp") + .thread(1) + .set_browsers( + Browsers::new() + .set_chrome() + .set_safari() + .set_edge() + .set_firefox() + .set_mozilla(), + ) + .build() + }) + .random() } From f5f0488954a19320a4ea024e7a29b4e23bf881ae Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 20:59:08 +0300 Subject: [PATCH 08/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20add?= =?UTF-8?q?=20several=20optimizations=20to=20the=20engine=20code=20(#180)(?= =?UTF-8?q?#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/engines/duckduckgo.rs | 60 +++++++++++++-------------------------- src/engines/searx.rs | 51 +++++++++++---------------------- 2 files changed, 36 insertions(+), 75 deletions(-) diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index 11b7d86..c716e8a 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -4,14 +4,14 @@ use std::collections::HashMap; -use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT}; +use reqwest::header::HeaderMap; use scraper::{Html, Selector}; use crate::results::aggregation_models::SearchResult; use super::engine_models::{EngineError, SearchEngine}; -use error_stack::{IntoReport, Report, Result, ResultExt}; +use error_stack::{Report, Result, ResultExt}; /// A new DuckDuckGo engine type defined in-order to implement the `SearchEngine` trait which allows to /// reduce code duplication as well as allows to create vector of different search engines easily. @@ -39,9 +39,9 @@ impl SearchEngine for DuckDuckGo { /// or HeaderMap fails to initialize. async fn results( &self, - query: String, + query: &str, page: u32, - user_agent: String, + user_agent: &str, request_timeout: u8, ) -> Result, EngineError> { // Page number can be missing or empty string and so appropriate handling is required @@ -61,38 +61,19 @@ impl SearchEngine for DuckDuckGo { }; // initializing HeaderMap and adding appropriate headers. - let mut header_map = HeaderMap::new(); - header_map.insert( - USER_AGENT, - user_agent - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); - header_map.insert( - REFERER, - "https://google.com/" - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); - header_map.insert( - CONTENT_TYPE, - "application/x-www-form-urlencoded" - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); - header_map.insert( - COOKIE, - "kl=wt-wt" - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); + let header_map = HeaderMap::try_from(&HashMap::from([ + ("USER_AGENT".to_string(), user_agent.to_string()), + ("REFERER".to_string(), "https://google.com/".to_string()), + ( + "CONTENT_TYPE".to_string(), + "application/x-www-form-urlencoded".to_string(), + ), + ("COOKIE".to_string(), "kl=wt-wt".to_string()), + ])) + .change_context(EngineError::UnexpectedError)?; let document: Html = Html::parse_document( - &DuckDuckGo::fetch_html_from_upstream(self, url, header_map, request_timeout).await?, + &DuckDuckGo::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?, ); let no_result: Selector = Selector::parse(".no-results") @@ -126,8 +107,7 @@ impl SearchEngine for DuckDuckGo { .next() .unwrap() .inner_html() - .trim() - .to_string(), + .trim(), format!( "https://{}", result @@ -136,15 +116,15 @@ impl SearchEngine for DuckDuckGo { .unwrap() .inner_html() .trim() - ), + ) + .as_str(), result .select(&result_desc) .next() .unwrap() .inner_html() - .trim() - .to_string(), - vec!["duckduckgo".to_string()], + .trim(), + &["duckduckgo"], ) }) .map(|search_result| (search_result.url.clone(), search_result)) diff --git a/src/engines/searx.rs b/src/engines/searx.rs index 4ad41f5..ca45cf0 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -2,14 +2,14 @@ //! by querying the upstream searx search engine instance with user provided query and with a page //! number if provided. -use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT}; +use reqwest::header::HeaderMap; use scraper::{Html, Selector}; use std::collections::HashMap; use crate::results::aggregation_models::SearchResult; use super::engine_models::{EngineError, SearchEngine}; -use error_stack::{IntoReport, Report, Result, ResultExt}; +use error_stack::{Report, Result, ResultExt}; /// A new Searx engine type defined in-order to implement the `SearchEngine` trait which allows to /// reduce code duplication as well as allows to create vector of different search engines easily. @@ -38,9 +38,9 @@ impl SearchEngine for Searx { async fn results( &self, - query: String, + query: &str, page: u32, - user_agent: String, + user_agent: &str, request_timeout: u8, ) -> Result, EngineError> { // Page number can be missing or empty string and so appropriate handling is required @@ -51,32 +51,16 @@ impl SearchEngine for Searx { }; // initializing headers and adding appropriate headers. - let mut header_map = HeaderMap::new(); - header_map.insert( - USER_AGENT, - user_agent - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); - header_map.insert( - REFERER, - "https://google.com/" - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); - header_map.insert( - CONTENT_TYPE, - "application/x-www-form-urlencoded" - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); - header_map.insert(COOKIE, "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".parse().into_report().change_context(EngineError::UnexpectedError)?); + let header_map = HeaderMap::try_from(&HashMap::from([ + ("USER_AGENT".to_string(), user_agent.to_string()), + ("REFERER".to_string(), "https://google.com/".to_string()), + ("CONTENT_TYPE".to_string(), "application/x-www-form-urlencoded".to_string()), + ("COOKIE".to_string(), "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".to_string()) + ])) + .change_context(EngineError::UnexpectedError)?; let document: Html = Html::parse_document( - &Searx::fetch_html_from_upstream(self, url, header_map, request_timeout).await?, + &Searx::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?, ); let no_result: Selector = Selector::parse("#urls>.dialog-error>p") @@ -117,24 +101,21 @@ impl SearchEngine for Searx { .next() .unwrap() .inner_html() - .trim() - .to_string(), + .trim(), result .select(&result_url) .next() .unwrap() .value() .attr("href") - .unwrap() - .to_string(), + .unwrap(), result .select(&result_desc) .next() .unwrap() .inner_html() - .trim() - .to_string(), - vec!["searx".to_string()], + .trim(), + &["searx"], ) }) .map(|search_result| (search_result.url.clone(), search_result)) From 2a68081ae23ad7d2183f51e2112584b7cc74bd8e Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 21:00:22 +0300 Subject: [PATCH 09/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20remov?= =?UTF-8?q?e=20deprecated=20intoreport=20functions=20&=20add=20minor=20opt?= =?UTF-8?q?imizations=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/engines/engine_models.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/engines/engine_models.rs b/src/engines/engine_models.rs index d33d13c..86fb207 100644 --- a/src/engines/engine_models.rs +++ b/src/engines/engine_models.rs @@ -2,7 +2,7 @@ //! the upstream search engines with the search query provided by the user. use crate::results::aggregation_models::SearchResult; -use error_stack::{IntoReport, Result, ResultExt}; +use error_stack::{Result, ResultExt}; use std::{collections::HashMap, fmt, time::Duration}; /// A custom error type used for handle engine associated errors. @@ -48,7 +48,7 @@ impl error_stack::Context for EngineError {} pub trait SearchEngine: Sync + Send { async fn fetch_html_from_upstream( &self, - url: String, + url: &str, header_map: reqwest::header::HeaderMap, request_timeout: u8, ) -> Result { @@ -59,19 +59,17 @@ pub trait SearchEngine: Sync + Send { .headers(header_map) // add spoofed headers to emulate human behavior .send() .await - .into_report() .change_context(EngineError::RequestError)? .text() .await - .into_report() .change_context(EngineError::RequestError)?) } async fn results( &self, - query: String, + query: &str, page: u32, - user_agent: String, + user_agent: &str, request_timeout: u8, ) -> Result, EngineError>; } From 2885f23ec96403fa51e443dd7aa0faad3f9173b1 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 21:02:23 +0300 Subject: [PATCH 10/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20repla?= =?UTF-8?q?ce=20vecs=20with=20smallvecs=20for=20smaller=20data=20sizes=20&?= =?UTF-8?q?=20replace=20to=5Fstrings=20with=20to=5Fowned=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/results/aggregation_models.rs | 60 +++++++++++++++---------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/results/aggregation_models.rs b/src/results/aggregation_models.rs index e985765..11b2e63 100644 --- a/src/results/aggregation_models.rs +++ b/src/results/aggregation_models.rs @@ -2,6 +2,7 @@ //! data scraped from the upstream search engines. use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; use crate::{config::parser_models::Style, engines::engine_models::EngineError}; @@ -16,13 +17,13 @@ use crate::{config::parser_models::Style, engines::engine_models::EngineError}; /// (href url in html in simple words). /// * `description` - The description of the search result. /// * `engine` - The names of the upstream engines from which this results were provided. -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] pub struct SearchResult { pub title: String, pub url: String, pub description: String, - pub engine: Vec, + pub engine: SmallVec<[String; 0]>, } impl SearchResult { @@ -35,12 +36,12 @@ impl SearchResult { /// (href url in html in simple words). /// * `description` - The description of the search result. /// * `engine` - The names of the upstream engines from which this results were provided. - pub fn new(title: String, url: String, description: String, engine: Vec) -> Self { + pub fn new(title: &str, url: &str, description: &str, engine: &[&str]) -> Self { SearchResult { - title, - url, - description, - engine, + title: title.to_owned(), + url: url.to_owned(), + description: description.to_owned(), + engine: engine.iter().map(|name| name.to_string()).collect(), } } @@ -49,8 +50,8 @@ impl SearchResult { /// # Arguments /// /// * `engine` - Takes an engine name provided as a String. - pub fn add_engines(&mut self, engine: String) { - self.engine.push(engine) + pub fn add_engines(&mut self, engine: &str) { + self.engine.push(engine.to_owned()) } /// A function which returns the engine name stored from the struct as a string. @@ -58,13 +59,12 @@ impl SearchResult { /// # Returns /// /// An engine name stored as a string from the struct. - pub fn engine(self) -> String { - self.engine.get(0).unwrap().to_string() + pub fn engine(&mut self) -> String { + std::mem::take(&mut self.engine[0]) } } -/// -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone)] pub struct EngineErrorInfo { pub error: String, pub engine: String, @@ -72,18 +72,18 @@ pub struct EngineErrorInfo { } impl EngineErrorInfo { - pub fn new(error: &EngineError, engine: String) -> Self { + pub fn new(error: &EngineError, engine: &str) -> Self { Self { error: match error { - EngineError::RequestError => String::from("RequestError"), - EngineError::EmptyResultSet => String::from("EmptyResultSet"), - EngineError::UnexpectedError => String::from("UnexpectedError"), + EngineError::RequestError => "RequestError".to_owned(), + EngineError::EmptyResultSet => "EmptyResultSet".to_owned(), + EngineError::UnexpectedError => "UnexpectedError".to_owned(), }, - engine, + engine: engine.to_owned(), severity_color: match error { - EngineError::RequestError => String::from("green"), - EngineError::EmptyResultSet => String::from("blue"), - EngineError::UnexpectedError => String::from("red"), + EngineError::RequestError => "green".to_owned(), + EngineError::EmptyResultSet => "blue".to_owned(), + EngineError::UnexpectedError => "red".to_owned(), }, } } @@ -108,7 +108,7 @@ pub struct SearchResults { pub results: Vec, pub page_query: String, pub style: Style, - pub engine_errors_info: Vec, + pub engine_errors_info: SmallVec<[EngineErrorInfo; 0]>, } impl SearchResults { @@ -124,19 +124,19 @@ impl SearchResults { /// given search query. pub fn new( results: Vec, - page_query: String, - engine_errors_info: Vec, + page_query: &str, + engine_errors_info: &[EngineErrorInfo], ) -> Self { - SearchResults { + Self { results, - page_query, - style: Style::new("".to_string(), "".to_string()), - engine_errors_info, + page_query: page_query.to_owned(), + style: Style::default(), + engine_errors_info: SmallVec::from(engine_errors_info), } } /// A setter function to add website style to the return search results. - pub fn add_style(&mut self, style: Style) { - self.style = style; + pub fn add_style(&mut self, style: &Style) { + self.style = style.to_owned(); } } From 13ce420642d826f57351a0e009c08bc1975acfa3 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 21:04:41 +0300 Subject: [PATCH 11/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20add?= =?UTF-8?q?=20several=20minor=20optimizations=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/results/aggregator.rs | 91 +++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index 3f06ecb..1942acc 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -64,14 +64,14 @@ type FutureVec = Vec, Report, + upstream_search_engines: &[EngineHandler], request_timeout: u8, ) -> Result> { - let user_agent: String = random_user_agent(); + let user_agent: &str = random_user_agent(); // Add a random delay before making the request. if random_delay || !debug { @@ -80,19 +80,18 @@ pub async fn aggregate( tokio::time::sleep(Duration::from_secs(delay_secs)).await; } - let mut names: Vec<&str> = vec![]; + let mut names: Vec<&str> = Vec::with_capacity(0); // create tasks for upstream result fetching let mut tasks: FutureVec = FutureVec::new(); for engine_handler in upstream_search_engines { - let (name, search_engine) = engine_handler.into_name_engine(); + let (name, search_engine) = engine_handler.to_owned().into_name_engine(); names.push(name); - let query: String = query.clone(); - let user_agent: String = user_agent.clone(); + let query: String = query.to_owned(); tasks.push(tokio::spawn(async move { search_engine - .results(query, page, user_agent.clone(), request_timeout) + .results(&query, page, user_agent, request_timeout) .await })); } @@ -110,7 +109,7 @@ pub async fn aggregate( let mut result_map: HashMap = HashMap::new(); let mut engine_errors_info: Vec = Vec::new(); - let mut handle_error = |error: Report, engine_name: String| { + let mut handle_error = |error: &Report, engine_name: &'static str| { log::error!("Engine Error: {:?}", error); engine_errors_info.push(EngineErrorInfo::new( error.downcast_ref::().unwrap(), @@ -120,7 +119,7 @@ pub async fn aggregate( for _ in 0..responses.len() { let response = responses.pop().unwrap(); - let engine = names.pop().unwrap().to_string(); + let engine = names.pop().unwrap(); if result_map.is_empty() { match response { @@ -128,7 +127,7 @@ pub async fn aggregate( result_map = results.clone(); } Err(error) => { - handle_error(error, engine); + handle_error(&error, engine); } } continue; @@ -140,13 +139,13 @@ pub async fn aggregate( result_map .entry(key) .and_modify(|result| { - result.add_engines(engine.clone()); + result.add_engines(engine); }) .or_insert_with(|| -> SearchResult { value }); }); } Err(error) => { - handle_error(error, engine); + handle_error(&error, engine); } } } @@ -155,24 +154,20 @@ pub async fn aggregate( filter_with_lists( &mut result_map, &mut blacklist_map, - &file_path(FileType::BlockList)?, + file_path(FileType::BlockList)?, )?; filter_with_lists( &mut blacklist_map, &mut result_map, - &file_path(FileType::AllowList)?, + file_path(FileType::AllowList)?, )?; drop(blacklist_map); let results: Vec = result_map.into_values().collect(); - Ok(SearchResults::new( - results, - query.to_string(), - engine_errors_info, - )) + Ok(SearchResults::new(results, query, &engine_errors_info)) } /// Filters a map of search results using a list of regex patterns. @@ -203,7 +198,10 @@ pub fn filter_with_lists( || re.is_match(&search_result.description.to_lowercase()) { // If the search result matches the regex pattern, move it from the original map to the resultant map - resultant_map.insert(url.clone(), map_to_be_filtered.remove(&url).unwrap()); + resultant_map.insert( + url.to_owned(), + map_to_be_filtered.remove(&url.to_owned()).unwrap(), + ); } } } @@ -214,6 +212,7 @@ pub fn filter_with_lists( #[cfg(test)] mod tests { use super::*; + use smallvec::smallvec; use std::collections::HashMap; use std::io::Write; use tempfile::NamedTempFile; @@ -223,22 +222,22 @@ mod tests { // Create a map of search results to filter let mut map_to_be_filtered = HashMap::new(); map_to_be_filtered.insert( - "https://www.example.com".to_string(), + "https://www.example.com".to_owned(), SearchResult { - title: "Example Domain".to_string(), - url: "https://www.example.com".to_string(), + title: "Example Domain".to_owned(), + url: "https://www.example.com".to_owned(), description: "This domain is for use in illustrative examples in documents." - .to_string(), - engine: vec!["Google".to_string(), "Bing".to_string()], + .to_owned(), + engine: smallvec!["Google".to_owned(), "Bing".to_owned()], }, ); map_to_be_filtered.insert( - "https://www.rust-lang.org/".to_string(), + "https://www.rust-lang.org/".to_owned(), SearchResult { - title: "Rust Programming Language".to_string(), - url: "https://www.rust-lang.org/".to_string(), - description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_string(), - engine: vec!["Google".to_string(), "DuckDuckGo".to_string()], + title: "Rust Programming Language".to_owned(), + url: "https://www.rust-lang.org/".to_owned(), + description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(), + engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()], }, ); @@ -267,22 +266,22 @@ mod tests { fn test_filter_with_lists_wildcard() -> Result<(), Box> { let mut map_to_be_filtered = HashMap::new(); map_to_be_filtered.insert( - "https://www.example.com".to_string(), + "https://www.example.com".to_owned(), SearchResult { - title: "Example Domain".to_string(), - url: "https://www.example.com".to_string(), + title: "Example Domain".to_owned(), + url: "https://www.example.com".to_owned(), description: "This domain is for use in illustrative examples in documents." - .to_string(), - engine: vec!["Google".to_string(), "Bing".to_string()], + .to_owned(), + engine: smallvec!["Google".to_owned(), "Bing".to_owned()], }, ); map_to_be_filtered.insert( - "https://www.rust-lang.org/".to_string(), + "https://www.rust-lang.org/".to_owned(), SearchResult { - title: "Rust Programming Language".to_string(), - url: "https://www.rust-lang.org/".to_string(), - description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_string(), - engine: vec!["Google".to_string(), "DuckDuckGo".to_string()], + title: "Rust Programming Language".to_owned(), + url: "https://www.rust-lang.org/".to_owned(), + description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(), + engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()], }, ); @@ -327,13 +326,13 @@ mod tests { fn test_filter_with_lists_invalid_regex() { let mut map_to_be_filtered = HashMap::new(); map_to_be_filtered.insert( - "https://www.example.com".to_string(), + "https://www.example.com".to_owned(), SearchResult { - title: "Example Domain".to_string(), - url: "https://www.example.com".to_string(), + title: "Example Domain".to_owned(), + url: "https://www.example.com".to_owned(), description: "This domain is for use in illustrative examples in documents." - .to_string(), - engine: vec!["Google".to_string(), "Bing".to_string()], + .to_owned(), + engine: smallvec!["Google".to_owned(), "Bing".to_owned()], }, ); From b2c72bdfc37407fd547dadefd127e413389a80dc Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 21:06:25 +0300 Subject: [PATCH 12/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20repla?= =?UTF-8?q?ce=20pass=20by=20value=20with=20pass=20by=20reference=20(#180)(?= =?UTF-8?q?#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/server/routes.rs | 46 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/server/routes.rs b/src/server/routes.rs index 8910f8f..2fca2bc 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -62,10 +62,10 @@ pub async fn not_found( /// * `engines` - It stores the user selected upstream search engines selected from the UI. #[allow(dead_code)] #[derive(Deserialize)] -struct Cookie { - theme: String, - colorscheme: String, - engines: Vec, +struct Cookie<'a> { + theme: &'a str, + colorscheme: &'a str, + engines: Vec<&'a str>, } /// Handles the route of search page of the `websurfx` meta search engine website and it takes @@ -111,9 +111,9 @@ pub async fn search( page - 1 ), &config, - query.to_string(), + query, page - 1, - req.clone(), + &req, ), results( format!( @@ -121,9 +121,9 @@ pub async fn search( config.binding_ip, config.port, query, page ), &config, - query.to_string(), + query, page, - req.clone(), + &req, ), results( format!( @@ -134,9 +134,9 @@ pub async fn search( page + 1 ), &config, - query.to_string(), + query, page + 1, - req.clone(), + &req, ) ); @@ -154,30 +154,28 @@ pub async fn search( async fn results( url: String, config: &Config, - query: String, + query: &str, page: u32, - req: HttpRequest, + req: &HttpRequest, ) -> Result> { //Initialize redis cache connection struct - let mut redis_cache = RedisCache::new(config.redis_url.clone())?; + let mut redis_cache = RedisCache::new(&config.redis_url, 5).await?; // fetch the cached results json. - let cached_results_json = redis_cache.cached_json(&url); + let cached_results_json = redis_cache.cached_json(&url).await; // check if fetched cache results was indeed fetched or it was an error and if so // handle the data accordingly. match cached_results_json { - Ok(results) => Ok(serde_json::from_str::(&results).unwrap()), + Ok(results) => Ok(serde_json::from_str::(&results)?), Err(_) => { // check if the cookie value is empty or not if it is empty then use the // default selected upstream search engines from the config file otherwise // parse the non-empty cookie and grab the user selected engines from the // UI and use that. - let mut results: crate::results::aggregation_models::SearchResults = match req - .cookie("appCookie") - { + let mut results: SearchResults = match req.cookie("appCookie") { Some(cookie_value) => { let cookie_value: Cookie = serde_json::from_str(cookie_value.name_value().1)?; - let engines = cookie_value + let engines: Vec = cookie_value .engines .iter() .filter_map(|name| EngineHandler::new(name)) @@ -188,7 +186,7 @@ async fn results( page, config.aggregator.random_delay, config.debug, - engines, + &engines, config.request_timeout, ) .await? @@ -199,14 +197,16 @@ async fn results( page, config.aggregator.random_delay, config.debug, - config.upstream_search_engines.clone(), + &config.upstream_search_engines, config.request_timeout, ) .await? } }; - results.add_style(config.style.clone()); - redis_cache.cache_results(serde_json::to_string(&results)?, &url)?; + results.add_style(&config.style); + redis_cache + .cache_results(&serde_json::to_string(&results)?, &url) + .await?; Ok(results) } } From fd6cb46f87e7e5f9618c58a9e811022f4fbcc448 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 21:07:57 +0300 Subject: [PATCH 13/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20repla?= =?UTF-8?q?ce=20string=20type=20to=20&str=20type=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index cd83d8a..e76344b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,7 +42,7 @@ use handler::paths::{file_path, FileType}; pub fn run(listener: TcpListener, config: Config) -> std::io::Result { let mut handlebars: Handlebars = Handlebars::new(); - let public_folder_path: String = file_path(FileType::Theme)?; + let public_folder_path: &str = file_path(FileType::Theme)?; handlebars .register_templates_directory(".html", format!("{}/templates", public_folder_path)) From 699e0ecaa97a1ebaae0c956f31de500029607dfb Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 21:08:50 +0300 Subject: [PATCH 14/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20updat?= =?UTF-8?q?e=20gitignore=20to=20ignore=20profiler=20results=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c8d5b9e..5889518 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ package-lock.json dump.rdb .vscode megalinter-reports/ +dhat-heap.json From 33f5b5e8ccae09cd725cacb6036c94254e81d71b Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 21:10:36 +0300 Subject: [PATCH 15/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20=20refactor:=20remov?= =?UTF-8?q?e=20&=20added=20some=20crates=20&=20add=20a=20new=20feature=20(?= =?UTF-8?q?#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 352 +++++++++++++++++++++++++++++++++++++---------------- Cargo.toml | 22 ++-- 2 files changed, 261 insertions(+), 113 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 412ae83..eecdba9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,7 +68,7 @@ dependencies = [ "actix-service", "actix-utils", "ahash 0.8.3", - "base64 0.21.2", + "base64 0.21.3", "bitflags 1.3.2", "brotli", "bytes 1.4.0", @@ -121,9 +121,9 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" +checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" dependencies = [ "futures-core", "tokio 1.32.0", @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e8613a75dd50cc45f473cee3c34d59ed677c0f7b44480ce3b8247d7dc519327" +checksum = "3eb13e7eef0423ea6eab0e59f6c72e7cb46d33691ad56a726b3cd07ddec2c2d4" dependencies = [ "actix-rt", "actix-service", @@ -141,8 +141,7 @@ dependencies = [ "futures-core", "futures-util", "mio 0.8.8", - "num_cpus", - "socket2 0.4.9", + "socket2 0.5.3", "tokio 1.32.0", "tracing", ] @@ -205,7 +204,7 @@ dependencies = [ "serde_urlencoded 0.7.1", "smallvec 1.11.0", "socket2 0.4.9", - "time 0.3.25", + "time 0.3.28", "url 2.4.0", ] @@ -223,9 +222,9 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -291,9 +290,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstyle" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" [[package]] name = "anyhow" @@ -301,6 +300,12 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + [[package]] name = "askama_escape" version = "0.10.3" @@ -335,9 +340,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", @@ -359,9 +364,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.2" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "bit-set" @@ -533,18 +538,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.23" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03aef18ddf7d879c15ce20f04826ef8418101c7e528014c3eeea13321047dca3" +checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.3.23" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ce6fffb678c9b80a70b6b6de0aad31df727623a70fd9a842c30cd573e2fa98" +checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6" dependencies = [ "anstyle", "clap_lex", @@ -552,9 +557,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "cloudabi" @@ -572,7 +577,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ "bytes 1.4.0", + "futures-core", "memchr", + "pin-project-lite", + "tokio 1.32.0", + "tokio-util", ] [[package]] @@ -598,7 +607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding 2.3.0", - "time 0.3.25", + "time 0.3.28", "version_check", ] @@ -833,6 +842,22 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "dhat" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2aaf837aaf456f6706cb46386ba8dffd4013a757e36f4ea05c20dd46b209a3" +dependencies = [ + "backtrace", + "lazy_static", + "mintex", + "parking_lot 0.12.1", + "rustc-hash", + "serde", + "serde_json", + "thousands", +] + [[package]] name = "digest" version = "0.10.7" @@ -878,9 +903,9 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if 1.0.0", ] @@ -931,9 +956,9 @@ dependencies = [ [[package]] name = "error-stack" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f00447f331c7f726db5b8532ebc9163519eed03c6d7c8b73c90b3ff5646ac85" +checksum = "e6a37ef405b504fc3b87a24fa52906d98cdd1a7d4e5ef2b49f0d5fead138fced" dependencies = [ "anyhow", "rustc_version 0.4.0", @@ -1062,6 +1087,21 @@ version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.28" @@ -1069,6 +1109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1083,10 +1124,38 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" dependencies = [ - "futures", + "futures 0.1.31", "num_cpus", ] +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", +] + [[package]] name = "futures-sink" version = "0.3.28" @@ -1105,10 +1174,16 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -1152,9 +1227,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "h2" @@ -1165,7 +1240,7 @@ dependencies = [ "byteorder", "bytes 0.4.12", "fnv", - "futures", + "futures 0.1.31", "http 0.1.21", "indexmap", "log", @@ -1283,7 +1358,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "http 0.1.21", "tokio-buf", ] @@ -1330,7 +1405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c843caf6296fc1f93444735205af9ed4e109a539005abb2564ae1d6fad34c52" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "futures-cpupool", "h2 0.1.26", "http 0.1.21", @@ -1384,7 +1459,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "hyper 0.12.36", "native-tls", "tokio-io", @@ -1691,6 +1766,16 @@ dependencies = [ "adler", ] +[[package]] +name = "mintex" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7c5ba1c3b5a23418d7bbf98c71c3d4946a0125002129231da8d6b723d559cb" +dependencies = [ + "once_cell", + "sys-info", +] + [[package]] name = "mio" version = "0.6.23" @@ -1734,6 +1819,20 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "mlua" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e" +dependencies = [ + "bstr", + "cc", + "num-traits", + "once_cell", + "pkg-config", + "rustc-hash", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -1796,9 +1895,9 @@ dependencies = [ [[package]] name = "object" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" dependencies = [ "memchr", ] @@ -1817,11 +1916,11 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.56" +version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "cfg-if 1.0.0", "foreign-types", "libc", @@ -1849,9 +1948,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.91" +version = "0.9.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac" +checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b" dependencies = [ "cc", "libc", @@ -2076,7 +2175,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" dependencies = [ - "siphasher 0.3.10", + "siphasher 0.3.11", ] [[package]] @@ -2085,14 +2184,34 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ - "siphasher 0.3.10", + "siphasher 0.3.11", +] + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] name = "pin-project-lite" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -2365,12 +2484,21 @@ version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffd6543a7bc6428396845f6854ccf3d1ae8823816592e2cbe74f20f50f209d02" dependencies = [ + "arc-swap", + "async-trait", + "bytes 1.4.0", "combine", + "futures 0.3.28", + "futures-util", "itoa 1.0.9", "percent-encoding 2.3.0", + "pin-project-lite", "ryu", "sha1_smol", "socket2 0.4.9", + "tokio 1.32.0", + "tokio-retry", + "tokio-util", "url 2.4.0", ] @@ -2391,9 +2519,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" dependencies = [ "aho-corasick", "memchr", @@ -2403,9 +2531,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" dependencies = [ "aho-corasick", "memchr", @@ -2414,9 +2542,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "reqwest" @@ -2430,7 +2558,7 @@ dependencies = [ "cookie_store", "encoding_rs", "flate2", - "futures", + "futures 0.1.31", "http 0.1.21", "hyper 0.12.36", "hyper-tls 0.3.2", @@ -2454,11 +2582,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.19" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20b9b67e2ca7dd9e9f9285b759de30ff538aab981abaaf7bc9bd90b84a0126c3" +checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "bytes 1.4.0", "encoding_rs", "futures-core", @@ -2489,36 +2617,18 @@ dependencies = [ "winreg 0.50.0", ] -[[package]] -name = "rlua" -version = "0.19.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d33e5ba15c3d43178f283ed5863d4531e292fc0e56fb773f3bea45f18e3a42a" -dependencies = [ - "bitflags 1.3.2", - "bstr", - "libc", - "num-traits", - "rlua-lua54-sys", -] - -[[package]] -name = "rlua-lua54-sys" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aafabafe1895cb4a2be81a56d7ff3d46bf4b5d2f9cfdbea2ed404cdabe96474" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "rustc-demangle" version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.2.3" @@ -2539,9 +2649,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" dependencies = [ "bitflags 2.4.0", "errno", @@ -2684,18 +2794,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.185" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.185" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc59dfdcbad1437773485e0367fea4b090a2e0a16d9ffc46af47764536a298ec" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", @@ -2791,9 +2901,9 @@ checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" [[package]] name = "siphasher" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "slab" @@ -2818,6 +2928,9 @@ name = "smallvec" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +dependencies = [ + "serde", +] [[package]] name = "socket2" @@ -2959,6 +3072,16 @@ dependencies = [ "unicode-xid 0.2.4", ] +[[package]] +name = "sys-info" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "tempfile" version = "3.8.0" @@ -3012,6 +3135,12 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "thousands" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" + [[package]] name = "time" version = "0.1.45" @@ -3025,9 +3154,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.25" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" +checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" dependencies = [ "deranged", "itoa 1.0.9", @@ -3044,9 +3173,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.11" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" +checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" dependencies = [ "time-core", ] @@ -3083,7 +3212,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "mio 0.6.23", "num_cpus", "tokio-current-thread", @@ -3122,7 +3251,7 @@ checksum = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46" dependencies = [ "bytes 0.4.12", "either", - "futures", + "futures 0.1.31", ] [[package]] @@ -3131,7 +3260,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" dependencies = [ - "futures", + "futures 0.1.31", "tokio-executor", ] @@ -3142,7 +3271,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" dependencies = [ "crossbeam-utils 0.7.2", - "futures", + "futures 0.1.31", ] [[package]] @@ -3152,7 +3281,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "log", ] @@ -3184,7 +3313,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" dependencies = [ "crossbeam-utils 0.7.2", - "futures", + "futures 0.1.31", "lazy_static", "log", "mio 0.6.23", @@ -3196,6 +3325,17 @@ dependencies = [ "tokio-sync", ] +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio 1.32.0", +] + [[package]] name = "tokio-sync" version = "0.1.8" @@ -3203,7 +3343,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" dependencies = [ "fnv", - "futures", + "futures 0.1.31", ] [[package]] @@ -3213,7 +3353,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "iovec", "mio 0.6.23", "tokio-io", @@ -3229,7 +3369,7 @@ dependencies = [ "crossbeam-deque 0.7.4", "crossbeam-queue", "crossbeam-utils 0.7.2", - "futures", + "futures 0.1.31", "lazy_static", "log", "num_cpus", @@ -3244,7 +3384,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" dependencies = [ "crossbeam-utils 0.7.2", - "futures", + "futures 0.1.31", "slab", "tokio-executor", ] @@ -3439,7 +3579,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230" dependencies = [ - "futures", + "futures 0.1.31", "log", "try-lock", ] @@ -3550,22 +3690,24 @@ dependencies = [ "actix-web", "async-trait", "criterion", + "dhat", "env_logger", "error-stack", "fake-useragent", + "futures 0.3.28", "handlebars", "log", "md5", - "once_cell", + "mlua", "rand 0.8.5", "redis", "regex", - "reqwest 0.11.19", - "rlua", + "reqwest 0.11.20", "rusty-hook", "scraper", "serde", "serde_json", + "smallvec 1.11.0", "tempfile", "tokio 1.32.0", ] diff --git a/Cargo.toml b/Cargo.toml index d36117b..cf5a114 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,9 +7,9 @@ repository = "https://github.com/neon-mmd/websurfx" license = "AGPL-3.0" [dependencies] -reqwest = {version="0.11.19",features=["json"]} -tokio = {version="1.32.0",features=["full"]} -serde = {version="1.0.185",features=["derive"]} +reqwest = {version="0.11.20",features=["json"]} +tokio = {version="1.32.0",features=["rt-multi-thread","macros"]} +serde = {version="1.0.188",features=["derive"]} handlebars = { version = "4.3.7", features = ["dir_source"] } scraper = {version="0.17.1"} actix-web = {version="4.3.1", features = ["cookies"]} @@ -19,14 +19,16 @@ serde_json = {version="1.0.105"} fake-useragent = {version="0.1.3"} env_logger = {version="0.10.0"} log = {version="0.4.20"} -rlua = {version="0.19.7"} -redis = {version="0.23.2"} +mlua = {version="0.8.10", features=["luajit"]} +redis = {version="0.23.2", features=["tokio-comp", "connection-manager"]} md5 = {version="0.7.0"} rand={version="0.8.5"} -once_cell = {version="1.18.0"} -error-stack = {version="0.3.1"} +error-stack = {version="0.4.0"} async-trait = {version="0.1.73"} regex = {version="1.9.3", features=["perf"]} +smallvec = {version="1.11.0", features=["union", "serde"]} +futures = {version="0.3.28"} +dhat = {version="0.3.2", optional = true} [dev-dependencies] rusty-hook = "^0.11.2" @@ -47,7 +49,8 @@ rpath = false [profile.release] opt-level = 3 -debug = false +debug = false # This should only be commented when testing with dhat profiler +# debug = 1 # This should only be uncommented when testing with dhat profiler split-debuginfo = '...' debug-assertions = false overflow-checks = false @@ -57,3 +60,6 @@ incremental = false codegen-units = 16 rpath = false strip = "debuginfo" + +[features] +dhat-heap = ["dep:dhat"] From 3db6c1017c08981932d3a811d84b553ef12a5420 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 21:12:55 +0300 Subject: [PATCH 16/31] =?UTF-8?q?=F0=9F=9A=80=20chore:=20bump=20the=20app?= =?UTF-8?q?=20version=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eecdba9..90092b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3683,7 +3683,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.18.1" +version = "0.18.2" dependencies = [ "actix-cors", "actix-files", diff --git a/Cargo.toml b/Cargo.toml index cf5a114..4ed6927 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.18.1" +version = "0.18.2" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" From fceacdb916e961e0e877fca33009b86f11b14f93 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 27 Aug 2023 21:17:21 +0300 Subject: [PATCH 17/31] =?UTF-8?q?=F0=9F=A7=B9=20chore:=20make=20cargo=20fo?= =?UTF-8?q?rmat=20happy=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/websurfx.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index 1f26af4..9aa5b69 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -6,7 +6,7 @@ use std::net::TcpListener; use websurfx::{config::parser::Config, run}; -/// A dhat heap memory profiler +/// A dhat heap memory profiler #[cfg(feature = "dhat-heap")] #[global_allocator] static ALLOC: dhat::Alloc = dhat::Alloc; @@ -22,7 +22,7 @@ async fn main() -> std::io::Result<()> { // A dhat heap profiler initialization. #[cfg(feature = "dhat-heap")] let _profiler = dhat::Profiler::new_heap(); - + // Initialize the parsed config file. let config = Config::parse(false).unwrap(); From 028463bb362df5a6b78b9a0880249aea45fb36f9 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 29 Aug 2023 20:07:28 +0300 Subject: [PATCH 18/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20refactor:=20replace?= =?UTF-8?q?=20jemalloc=20with=20mimalloc=20(#178)(#180)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/websurfx.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index 9aa5b69..bc8e7ce 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -3,6 +3,7 @@ //! This module contains the main function which handles the logging of the application to the //! stdout and handles the command line arguments provided and launches the `websurfx` server. +use mimalloc::MiMalloc; use std::net::TcpListener; use websurfx::{config::parser::Config, run}; @@ -11,6 +12,10 @@ use websurfx::{config::parser::Config, run}; #[global_allocator] static ALLOC: dhat::Alloc = dhat::Alloc; +#[cfg(not(feature = "dhat-heap"))] +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + /// The function that launches the main server and registers all the routes of the website. /// /// # Error From 1de52decd39ee46f69ea36ef7665c21d643b2338 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 29 Aug 2023 20:10:32 +0300 Subject: [PATCH 19/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20refactor:=20add=20cl?= =?UTF-8?q?one=20trait=20to=20`RedisCache`=20struct=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cache/cacher.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cache/cacher.rs b/src/cache/cacher.rs index 6932dea..b2508b5 100644 --- a/src/cache/cacher.rs +++ b/src/cache/cacher.rs @@ -17,6 +17,7 @@ use super::error::PoolError; /// * `pool_size` - It stores the size of the connection pool (in other words the number of /// connections that should be stored in the pool). /// * `current_connection` - It stores the index of which connection is being used at the moment. +#[derive(Clone)] pub struct RedisCache { connection_pool: Vec, pool_size: u8, From 7b1f93b232f7ed8a1fb82a3cbe9fea4c6181910b Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 29 Aug 2023 20:12:15 +0300 Subject: [PATCH 20/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20refactor:=20add=20co?= =?UTF-8?q?mpress=20middleware=20to=20reduce=20memory=20usage=20(#180)(#17?= =?UTF-8?q?8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index e76344b..97bff01 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,7 +14,12 @@ use crate::server::routes; use actix_cors::Cors; use actix_files as fs; -use actix_web::{dev::Server, http::header, middleware::Logger, web, App, HttpServer}; +use actix_web::{ + dev::Server, + http::header, + middleware::{Compress, Logger}, + web, App, HttpServer, +}; use config::parser::Config; use handlebars::Handlebars; use handler::paths::{file_path, FileType}; @@ -68,6 +73,7 @@ pub fn run(listener: TcpListener, config: Config) -> std::io::Result { .app_data(web::Data::new(config.clone())) .wrap(cors) .wrap(Logger::default()) // added logging middleware for logging. + .wrap(Compress::default()) // compress request headers to reduce memory usage. // Serve images and static files (css and js files). .service( fs::Files::new("/static", format!("{}/static", public_folder_path)) From 4c298ce18c3aeed7b6a8b9fade8ad7dbd931531e Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 29 Aug 2023 20:15:06 +0300 Subject: [PATCH 21/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20refactor:=20add=20co?= =?UTF-8?q?de=20to=20initialize=20redis=20cache=20struct=20only=20once=20(?= =?UTF-8?q?#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/server/routes.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/server/routes.rs b/src/server/routes.rs index 2fca2bc..8b4a523 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -16,6 +16,10 @@ use handlebars::Handlebars; use serde::Deserialize; use tokio::join; +// ---- Constants ---- +/// Initialize redis cache connection once and store it on the heap. +const REDIS_CACHE: async_once_cell::OnceCell = async_once_cell::OnceCell::new(); + /// A named struct which deserializes all the user provided search parameters and stores them. /// /// # Fields @@ -158,10 +162,17 @@ async fn results( page: u32, req: &HttpRequest, ) -> Result> { - //Initialize redis cache connection struct - let mut redis_cache = RedisCache::new(&config.redis_url, 5).await?; + let redis_cache: RedisCache = REDIS_CACHE + .get_or_init(async { + // Initialize redis cache connection pool only one and store it in the heap. + RedisCache::new(&config.redis_url, 5).await.unwrap() + }) + .await + .clone(); + // fetch the cached results json. - let cached_results_json = redis_cache.cached_json(&url).await; + let cached_results_json: Result> = + redis_cache.clone().cached_json(&url).await; // check if fetched cache results was indeed fetched or it was an error and if so // handle the data accordingly. match cached_results_json { @@ -205,6 +216,7 @@ async fn results( }; results.add_style(&config.style); redis_cache + .clone() .cache_results(&serde_json::to_string(&results)?, &url) .await?; Ok(results) From 4157ba8b7fe9039696f6d0f1e7f114b0f5976536 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 29 Aug 2023 20:16:51 +0300 Subject: [PATCH 22/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20refactor:=20add=20ne?= =?UTF-8?q?w=20crates=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 27 +++++++++++++++++++++++++++ Cargo.toml | 4 +++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 90092b4..5389dea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -312,6 +312,12 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" +[[package]] +name = "async-once-cell" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9338790e78aa95a416786ec8389546c4b6a1dfc3dc36071ed9518a9413a542eb" + [[package]] name = "async-trait" version = "0.1.73" @@ -1613,6 +1619,16 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "libmimalloc-sys" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d058a81af0d1c22d7a1c948576bee6d673f7af3c0f35564abd6c81122f513d" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "linux-raw-sys" version = "0.4.5" @@ -1741,6 +1757,15 @@ dependencies = [ "autocfg 1.1.0", ] +[[package]] +name = "mimalloc" +version = "0.1.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972e5f23f6716f62665760b0f4cbf592576a80c7b879ba9beaafc0e558894127" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.17" @@ -3688,6 +3713,7 @@ dependencies = [ "actix-cors", "actix-files", "actix-web", + "async-once-cell", "async-trait", "criterion", "dhat", @@ -3698,6 +3724,7 @@ dependencies = [ "handlebars", "log", "md5", + "mimalloc", "mlua", "rand 0.8.5", "redis", diff --git a/Cargo.toml b/Cargo.toml index 4ed6927..aa075b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,10 +25,12 @@ md5 = {version="0.7.0"} rand={version="0.8.5"} error-stack = {version="0.4.0"} async-trait = {version="0.1.73"} -regex = {version="1.9.3", features=["perf"]} +regex = {version="1.9.4", features=["perf"]} smallvec = {version="1.11.0", features=["union", "serde"]} futures = {version="0.3.28"} dhat = {version="0.3.2", optional = true} +mimalloc = { version = "0.1.38", default-features = false } +async-once-cell = {version="0.5.3"} [dev-dependencies] rusty-hook = "^0.11.2" From b508e6009f5b6165a99505aac6e87404831de91a Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 30 Aug 2023 13:14:53 +0300 Subject: [PATCH 23/31] =?UTF-8?q?=F0=9F=9A=80=20chore:=20bump=20the=20app?= =?UTF-8?q?=20version=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 7fd9246ee45eba283af49c7ec73434ff6f4c39a7 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 30 Aug 2023 13:18:31 +0300 Subject: [PATCH 24/31] =?UTF-8?q?=F0=9F=9A=80=20chore:=20bump=20the=20app?= =?UTF-8?q?=20version=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 181 +++++++++++++++++++++++++---------------------------- Cargo.toml | 2 +- 2 files changed, 85 insertions(+), 98 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 412ae83..be881be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,17 +59,17 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.3.1" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2079246596c18b4a33e274ae10c0e50613f4d32a4198e09c7b93771013fed74" +checksum = "a92ef85799cba03f76e4f7c10f533e66d87c9a7e7055f3391f09000ad8351bc9" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-utils", - "ahash 0.8.3", - "base64 0.21.2", - "bitflags 1.3.2", + "ahash", + "base64 0.21.3", + "bitflags 2.4.0", "brotli", "bytes 1.4.0", "bytestring", @@ -121,9 +121,9 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" +checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" dependencies = [ "futures-core", "tokio 1.32.0", @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e8613a75dd50cc45f473cee3c34d59ed677c0f7b44480ce3b8247d7dc519327" +checksum = "3eb13e7eef0423ea6eab0e59f6c72e7cb46d33691ad56a726b3cd07ddec2c2d4" dependencies = [ "actix-rt", "actix-service", @@ -141,8 +141,7 @@ dependencies = [ "futures-core", "futures-util", "mio 0.8.8", - "num_cpus", - "socket2 0.4.9", + "socket2 0.5.3", "tokio 1.32.0", "tracing", ] @@ -170,9 +169,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.3.1" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3cb42f9566ab176e1ef0b8b3a896529062b4efc6be0123046095914c4c1c96" +checksum = "0e4a5b5e29603ca8c94a77c65cf874718ceb60292c5a5c3e5f4ace041af462b9" dependencies = [ "actix-codec", "actix-http", @@ -183,7 +182,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web-codegen", - "ahash 0.7.6", + "ahash", "bytes 1.4.0", "bytestring", "cfg-if 1.0.0", @@ -192,7 +191,6 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "http 0.2.9", "itoa 1.0.9", "language-tags", "log", @@ -204,28 +202,28 @@ dependencies = [ "serde_json", "serde_urlencoded 0.7.1", "smallvec 1.11.0", - "socket2 0.4.9", - "time 0.3.25", - "url 2.4.0", + "socket2 0.5.3", + "time 0.3.28", + "url 2.4.1", ] [[package]] name = "actix-web-codegen" -version = "4.2.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" +checksum = "eb1f50ebbb30eca122b188319a4398b3f7bb4a8cdf50ecfb73bfc6a3c3ce54f5" dependencies = [ "actix-router", "proc-macro2 1.0.66", "quote 1.0.33", - "syn 1.0.109", + "syn 2.0.29", ] [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -236,17 +234,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "ahash" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" -dependencies = [ - "getrandom", - "once_cell", - "version_check", -] - [[package]] name = "ahash" version = "0.8.3" @@ -261,9 +248,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" dependencies = [ "memchr", ] @@ -291,9 +278,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstyle" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" [[package]] name = "anyhow" @@ -335,9 +322,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", @@ -359,9 +346,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.2" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "bit-set" @@ -533,18 +520,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.23" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03aef18ddf7d879c15ce20f04826ef8418101c7e528014c3eeea13321047dca3" +checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.3.23" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ce6fffb678c9b80a70b6b6de0aad31df727623a70fd9a842c30cd573e2fa98" +checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d" dependencies = [ "anstyle", "clap_lex", @@ -552,9 +539,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "cloudabi" @@ -598,7 +585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding 2.3.0", - "time 0.3.25", + "time 0.3.28", "version_check", ] @@ -878,9 +865,9 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if 1.0.0", ] @@ -910,9 +897,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", @@ -1152,9 +1139,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "h2" @@ -1644,9 +1631,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f478948fd84d9f8e86967bf432640e46adfb5a4bd4f14ef7e864ab38220534ae" [[package]] name = "memoffset" @@ -1796,9 +1783,9 @@ dependencies = [ [[package]] name = "object" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" dependencies = [ "memchr", ] @@ -1817,11 +1804,11 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.56" +version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "cfg-if 1.0.0", "foreign-types", "libc", @@ -1849,9 +1836,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.91" +version = "0.9.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac" +checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b" dependencies = [ "cc", "libc", @@ -2076,7 +2063,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" dependencies = [ - "siphasher 0.3.10", + "siphasher 0.3.11", ] [[package]] @@ -2085,14 +2072,14 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ - "siphasher 0.3.10", + "siphasher 0.3.11", ] [[package]] name = "pin-project-lite" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -2171,7 +2158,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95b4ce31ff0a27d93c8de1849cf58162283752f065a90d508f1105fa6c9a213f" dependencies = [ "idna 0.2.3", - "url 2.4.0", + "url 2.4.1", ] [[package]] @@ -2371,7 +2358,7 @@ dependencies = [ "ryu", "sha1_smol", "socket2 0.4.9", - "url 2.4.0", + "url 2.4.1", ] [[package]] @@ -2391,9 +2378,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" dependencies = [ "aho-corasick", "memchr", @@ -2403,9 +2390,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" dependencies = [ "aho-corasick", "memchr", @@ -2414,9 +2401,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "reqwest" @@ -2454,11 +2441,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.19" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20b9b67e2ca7dd9e9f9285b759de30ff538aab981abaaf7bc9bd90b84a0126c3" +checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "bytes 1.4.0", "encoding_rs", "futures-core", @@ -2482,7 +2469,7 @@ dependencies = [ "tokio 1.32.0", "tokio-native-tls", "tower-service", - "url 2.4.0", + "url 2.4.1", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -2539,9 +2526,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "ed6248e1caa625eb708e266e06159f135e8c26f2bb7ceb72dc4b2766d0340964" dependencies = [ "bitflags 2.4.0", "errno", @@ -2598,7 +2585,7 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c95a930e03325234c18c7071fd2b60118307e025d6fff3e12745ffbf63a3d29c" dependencies = [ - "ahash 0.8.3", + "ahash", "cssparser", "ego-tree", "getopts", @@ -2684,18 +2671,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.185" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.185" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc59dfdcbad1437773485e0367fea4b090a2e0a16d9ffc46af47764536a298ec" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", @@ -2791,9 +2778,9 @@ checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" [[package]] name = "siphasher" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "slab" @@ -3025,9 +3012,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.25" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" +checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" dependencies = [ "deranged", "itoa 1.0.9", @@ -3044,9 +3031,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.11" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" +checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" dependencies = [ "time-core", ] @@ -3387,9 +3374,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna 0.4.0", @@ -3543,7 +3530,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.18.1" +version = "0.18.4" dependencies = [ "actix-cors", "actix-files", @@ -3560,7 +3547,7 @@ dependencies = [ "rand 0.8.5", "redis", "regex", - "reqwest 0.11.19", + "reqwest 0.11.20", "rlua", "rusty-hook", "scraper", diff --git a/Cargo.toml b/Cargo.toml index d36117b..ca01dff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.18.1" +version = "0.18.4" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" From 5f20b316164a02fbc76f22ad60636e31ab7735da Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 30 Aug 2023 13:22:54 +0300 Subject: [PATCH 25/31] =?UTF-8?q?=F0=9F=9A=80=20chore:=20bump=20the=20app?= =?UTF-8?q?=20version=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 74 +++++++++++++++++++++++------------------------------- Cargo.toml | 2 +- 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 90092b4..0ceb626 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,17 +59,17 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.3.1" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2079246596c18b4a33e274ae10c0e50613f4d32a4198e09c7b93771013fed74" +checksum = "a92ef85799cba03f76e4f7c10f533e66d87c9a7e7055f3391f09000ad8351bc9" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-utils", - "ahash 0.8.3", + "ahash", "base64 0.21.3", - "bitflags 1.3.2", + "bitflags 2.4.0", "brotli", "bytes 1.4.0", "bytestring", @@ -169,9 +169,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.3.1" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3cb42f9566ab176e1ef0b8b3a896529062b4efc6be0123046095914c4c1c96" +checksum = "0e4a5b5e29603ca8c94a77c65cf874718ceb60292c5a5c3e5f4ace041af462b9" dependencies = [ "actix-codec", "actix-http", @@ -182,7 +182,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web-codegen", - "ahash 0.7.6", + "ahash", "bytes 1.4.0", "bytestring", "cfg-if 1.0.0", @@ -191,7 +191,6 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "http 0.2.9", "itoa 1.0.9", "language-tags", "log", @@ -203,21 +202,21 @@ dependencies = [ "serde_json", "serde_urlencoded 0.7.1", "smallvec 1.11.0", - "socket2 0.4.9", + "socket2 0.5.3", "time 0.3.28", - "url 2.4.0", + "url 2.4.1", ] [[package]] name = "actix-web-codegen" -version = "4.2.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" +checksum = "eb1f50ebbb30eca122b188319a4398b3f7bb4a8cdf50ecfb73bfc6a3c3ce54f5" dependencies = [ "actix-router", "proc-macro2 1.0.66", "quote 1.0.33", - "syn 1.0.109", + "syn 2.0.29", ] [[package]] @@ -235,17 +234,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "ahash" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" -dependencies = [ - "getrandom", - "once_cell", - "version_check", -] - [[package]] name = "ahash" version = "0.8.3" @@ -260,9 +248,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" dependencies = [ "memchr", ] @@ -538,18 +526,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.0" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d" +checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.4.0" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6" +checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d" dependencies = [ "anstyle", "clap_lex", @@ -935,9 +923,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", @@ -1719,9 +1707,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f478948fd84d9f8e86967bf432640e46adfb5a4bd4f14ef7e864ab38220534ae" [[package]] name = "memoffset" @@ -2290,7 +2278,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95b4ce31ff0a27d93c8de1849cf58162283752f065a90d508f1105fa6c9a213f" dependencies = [ "idna 0.2.3", - "url 2.4.0", + "url 2.4.1", ] [[package]] @@ -2499,7 +2487,7 @@ dependencies = [ "tokio 1.32.0", "tokio-retry", "tokio-util", - "url 2.4.0", + "url 2.4.1", ] [[package]] @@ -2610,7 +2598,7 @@ dependencies = [ "tokio 1.32.0", "tokio-native-tls", "tower-service", - "url 2.4.0", + "url 2.4.1", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -2649,9 +2637,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.9" +version = "0.38.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" +checksum = "ed6248e1caa625eb708e266e06159f135e8c26f2bb7ceb72dc4b2766d0340964" dependencies = [ "bitflags 2.4.0", "errno", @@ -2708,7 +2696,7 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c95a930e03325234c18c7071fd2b60118307e025d6fff3e12745ffbf63a3d29c" dependencies = [ - "ahash 0.8.3", + "ahash", "cssparser", "ego-tree", "getopts", @@ -3527,9 +3515,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna 0.4.0", @@ -3683,7 +3671,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.18.2" +version = "0.18.5" dependencies = [ "actix-cors", "actix-files", diff --git a/Cargo.toml b/Cargo.toml index 4ed6927..04cb92f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.18.2" +version = "0.18.5" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" From a799af312525cb198fd744826ca4118f26bef3c8 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 30 Aug 2023 18:09:58 +0300 Subject: [PATCH 26/31] =?UTF-8?q?=F0=9F=A7=B9=20chore:=20add=20fixes=20to?= =?UTF-8?q?=20code=20&=20make=20rustfmt=20&=20clippy=20happy=20(#180)(#178?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 181 +++++++++++++++++++++++++++++--------- Cargo.toml | 13 +-- src/bin/websurfx.rs | 4 +- src/lib.rs | 2 +- src/results/aggregator.rs | 9 +- src/server/routes.rs | 8 +- 6 files changed, 158 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be881be..d2f40f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -288,6 +288,12 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + [[package]] name = "askama_escape" version = "0.10.3" @@ -559,7 +565,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ "bytes 1.4.0", + "futures-core", "memchr", + "pin-project-lite", + "tokio 1.32.0", + "tokio-util", ] [[package]] @@ -1049,6 +1059,21 @@ version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.28" @@ -1056,6 +1081,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1070,10 +1096,38 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" dependencies = [ - "futures", + "futures 0.1.31", "num_cpus", ] +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", +] + [[package]] name = "futures-sink" version = "0.3.28" @@ -1092,10 +1146,16 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -1152,7 +1212,7 @@ dependencies = [ "byteorder", "bytes 0.4.12", "fnv", - "futures", + "futures 0.1.31", "http 0.1.21", "indexmap", "log", @@ -1270,7 +1330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "http 0.1.21", "tokio-buf", ] @@ -1317,7 +1377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c843caf6296fc1f93444735205af9ed4e109a539005abb2564ae1d6fad34c52" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "futures-cpupool", "h2 0.1.26", "http 0.1.21", @@ -1371,7 +1431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "hyper 0.12.36", "native-tls", "tokio-io", @@ -1721,6 +1781,20 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "mlua" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e" +dependencies = [ + "bstr", + "cc", + "num-traits", + "once_cell", + "pkg-config", + "rustc-hash", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -2075,6 +2149,26 @@ dependencies = [ "siphasher 0.3.11", ] +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -2352,12 +2446,21 @@ version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffd6543a7bc6428396845f6854ccf3d1ae8823816592e2cbe74f20f50f209d02" dependencies = [ + "arc-swap", + "async-trait", + "bytes 1.4.0", "combine", + "futures 0.3.28", + "futures-util", "itoa 1.0.9", "percent-encoding 2.3.0", + "pin-project-lite", "ryu", "sha1_smol", "socket2 0.4.9", + "tokio 1.32.0", + "tokio-retry", + "tokio-util", "url 2.4.1", ] @@ -2417,7 +2520,7 @@ dependencies = [ "cookie_store", "encoding_rs", "flate2", - "futures", + "futures 0.1.31", "http 0.1.21", "hyper 0.12.36", "hyper-tls 0.3.2", @@ -2476,36 +2579,18 @@ dependencies = [ "winreg 0.50.0", ] -[[package]] -name = "rlua" -version = "0.19.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d33e5ba15c3d43178f283ed5863d4531e292fc0e56fb773f3bea45f18e3a42a" -dependencies = [ - "bitflags 1.3.2", - "bstr", - "libc", - "num-traits", - "rlua-lua54-sys", -] - -[[package]] -name = "rlua-lua54-sys" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aafabafe1895cb4a2be81a56d7ff3d46bf4b5d2f9cfdbea2ed404cdabe96474" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "rustc-demangle" version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.2.3" @@ -3070,7 +3155,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "mio 0.6.23", "num_cpus", "tokio-current-thread", @@ -3109,7 +3194,7 @@ checksum = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46" dependencies = [ "bytes 0.4.12", "either", - "futures", + "futures 0.1.31", ] [[package]] @@ -3118,7 +3203,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" dependencies = [ - "futures", + "futures 0.1.31", "tokio-executor", ] @@ -3129,7 +3214,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" dependencies = [ "crossbeam-utils 0.7.2", - "futures", + "futures 0.1.31", ] [[package]] @@ -3139,7 +3224,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "log", ] @@ -3171,7 +3256,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" dependencies = [ "crossbeam-utils 0.7.2", - "futures", + "futures 0.1.31", "lazy_static", "log", "mio 0.6.23", @@ -3183,6 +3268,17 @@ dependencies = [ "tokio-sync", ] +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio 1.32.0", +] + [[package]] name = "tokio-sync" version = "0.1.8" @@ -3190,7 +3286,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" dependencies = [ "fnv", - "futures", + "futures 0.1.31", ] [[package]] @@ -3200,7 +3296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "iovec", "mio 0.6.23", "tokio-io", @@ -3216,7 +3312,7 @@ dependencies = [ "crossbeam-deque 0.7.4", "crossbeam-queue", "crossbeam-utils 0.7.2", - "futures", + "futures 0.1.31", "lazy_static", "log", "num_cpus", @@ -3231,7 +3327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" dependencies = [ "crossbeam-utils 0.7.2", - "futures", + "futures 0.1.31", "slab", "tokio-executor", ] @@ -3426,7 +3522,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230" dependencies = [ - "futures", + "futures 0.1.31", "log", "try-lock", ] @@ -3540,15 +3636,16 @@ dependencies = [ "env_logger", "error-stack", "fake-useragent", + "futures 0.3.28", "handlebars", "log", "md5", + "mlua", "once_cell", "rand 0.8.5", "redis", "regex", "reqwest 0.11.20", - "rlua", "rusty-hook", "scraper", "serde", diff --git a/Cargo.toml b/Cargo.toml index ca01dff..b7199b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,26 +7,27 @@ repository = "https://github.com/neon-mmd/websurfx" license = "AGPL-3.0" [dependencies] -reqwest = {version="0.11.19",features=["json"]} +reqwest = {version="0.11.20",features=["json"]} tokio = {version="1.32.0",features=["full"]} -serde = {version="1.0.185",features=["derive"]} +serde = {version="1.0.188",features=["derive"]} handlebars = { version = "4.3.7", features = ["dir_source"] } scraper = {version="0.17.1"} -actix-web = {version="4.3.1", features = ["cookies"]} +actix-web = {version="4.4.0", features = ["cookies"]} actix-files = {version="0.6.2"} actix-cors = {version="0.6.4"} serde_json = {version="1.0.105"} fake-useragent = {version="0.1.3"} env_logger = {version="0.10.0"} log = {version="0.4.20"} -rlua = {version="0.19.7"} -redis = {version="0.23.2"} +mlua = {version="0.8.10",features=["luajit"]} +redis = {version="0.23.2",features=["tokio-comp","connection-manager"]} md5 = {version="0.7.0"} rand={version="0.8.5"} once_cell = {version="1.18.0"} error-stack = {version="0.3.1"} async-trait = {version="0.1.73"} -regex = {version="1.9.3", features=["perf"]} +regex = {version="1.9.4", features=["perf"]} +futures = {version="0.3.28"} [dev-dependencies] rusty-hook = "^0.11.2" diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index 1f26af4..9aa5b69 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -6,7 +6,7 @@ use std::net::TcpListener; use websurfx::{config::parser::Config, run}; -/// A dhat heap memory profiler +/// A dhat heap memory profiler #[cfg(feature = "dhat-heap")] #[global_allocator] static ALLOC: dhat::Alloc = dhat::Alloc; @@ -22,7 +22,7 @@ async fn main() -> std::io::Result<()> { // A dhat heap profiler initialization. #[cfg(feature = "dhat-heap")] let _profiler = dhat::Profiler::new_heap(); - + // Initialize the parsed config file. let config = Config::parse(false).unwrap(); diff --git a/src/lib.rs b/src/lib.rs index cd83d8a..e76344b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,7 +42,7 @@ use handler::paths::{file_path, FileType}; pub fn run(listener: TcpListener, config: Config) -> std::io::Result { let mut handlebars: Handlebars = Handlebars::new(); - let public_folder_path: String = file_path(FileType::Theme)?; + let public_folder_path: &str = file_path(FileType::Theme)?; handlebars .register_templates_directory(".html", format!("{}/templates", public_folder_path)) diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index 3f06ecb..f9bb642 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -71,7 +71,7 @@ pub async fn aggregate( upstream_search_engines: Vec, request_timeout: u8, ) -> Result> { - let user_agent: String = random_user_agent(); + let user_agent: &str = random_user_agent(); // Add a random delay before making the request. if random_delay || !debug { @@ -89,10 +89,9 @@ pub async fn aggregate( let (name, search_engine) = engine_handler.into_name_engine(); names.push(name); let query: String = query.clone(); - let user_agent: String = user_agent.clone(); tasks.push(tokio::spawn(async move { search_engine - .results(query, page, user_agent.clone(), request_timeout) + .results(query, page, user_agent.to_owned(), request_timeout) .await })); } @@ -155,13 +154,13 @@ pub async fn aggregate( filter_with_lists( &mut result_map, &mut blacklist_map, - &file_path(FileType::BlockList)?, + file_path(FileType::BlockList)?, )?; filter_with_lists( &mut blacklist_map, &mut result_map, - &file_path(FileType::AllowList)?, + file_path(FileType::AllowList)?, )?; drop(blacklist_map); diff --git a/src/server/routes.rs b/src/server/routes.rs index 8910f8f..f00cced 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -159,9 +159,9 @@ async fn results( req: HttpRequest, ) -> Result> { //Initialize redis cache connection struct - let mut redis_cache = RedisCache::new(config.redis_url.clone())?; + let mut redis_cache = RedisCache::new(&config.redis_url, 5).await?; // fetch the cached results json. - let cached_results_json = redis_cache.cached_json(&url); + let cached_results_json = redis_cache.cached_json(&url).await; // check if fetched cache results was indeed fetched or it was an error and if so // handle the data accordingly. match cached_results_json { @@ -206,7 +206,9 @@ async fn results( } }; results.add_style(config.style.clone()); - redis_cache.cache_results(serde_json::to_string(&results)?, &url)?; + redis_cache + .cache_results(&serde_json::to_string(&results)?, &url) + .await?; Ok(results) } } From 55a7125ceae8a17e0ad4359bf205bf8f7f62b28b Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 30 Aug 2023 19:05:38 +0300 Subject: [PATCH 27/31] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20refactor:=20add=20bu?= =?UTF-8?q?ild=20time=20binary=20file=20optimization=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 2 +- Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 348340a..b642f3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,7 +67,7 @@ dependencies = [ "actix-rt", "actix-service", "actix-utils", - "ahash 0.8.3", + "ahash", "base64 0.21.3", "bitflags 2.4.0", "brotli", diff --git a/Cargo.toml b/Cargo.toml index 321891b..4c67969 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,10 +56,10 @@ debug = false # This should only be commented when testing with dhat profiler split-debuginfo = '...' debug-assertions = false overflow-checks = false -lto = 'thin' +lto = true panic = 'abort' incremental = false -codegen-units = 16 +codegen-units = 1 rpath = false strip = "debuginfo" From 5dc4d98e5a55b09862969221b74afa6f29b68090 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Thu, 7 Sep 2023 18:42:25 +0100 Subject: [PATCH 28/31] =?UTF-8?q?=F0=9F=93=9D=20Fix=20link=20to=20Document?= =?UTF-8?q?ation=20in=20Readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 07d73f5..8287c1d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Readme | Discord | GitHub | - Documentation + Documentation

Date: Fri, 8 Sep 2023 23:52:37 +0800 Subject: [PATCH 29/31] Bump the version of Debian distroless container --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0c54453..2611aca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ COPY . . RUN cargo install --path . # We do not need the Rust toolchain to run the binary! -FROM gcr.io/distroless/cc-debian11 +FROM gcr.io/distroless/cc-debian12 COPY --from=builder /app/public/ /opt/websurfx/public/ COPY --from=builder /app/websurfx/config.lua /etc/xdg/websurfx/config.lua COPY --from=builder /usr/local/cargo/bin/* /usr/local/bin/ From adcebb57de0e5f482759ab28d682c1cde6242b9d Mon Sep 17 00:00:00 2001 From: Siddharth Tiwari Date: Sun, 10 Sep 2023 09:36:44 +0530 Subject: [PATCH 30/31] fixed the issue --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 8f2ee2e..54d4355 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -109,7 +109,7 @@ colorscheme = "catppuccin-mocha" -- the colorscheme name which should be used fo theme = "simple" -- the theme name which should be used for the website -- ### Caching ### -redis_connection_url = "redis://redis:6379" -- redis connection url address on which the client should connect on. +redis_url = "redis://redis:6379" -- redis connection url address on which the client should connect on. -- ### Search Engines ### upstream_search_engines = { DuckDuckGo = true, Searx = false } -- select the upstream search engines from which the results should be fetched. From d14a5d4e4d39acbb5aaae42619037eae913bc6aa Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 10 Sep 2023 18:02:04 +0300 Subject: [PATCH 31/31] =?UTF-8?q?=F0=9F=A7=B9=20chore:=20make=20github=20a?= =?UTF-8?q?ctions=20happy=20(#180)(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c8dcb55..35cf219 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ once_cell = {version="1.18.0"} error-stack = {version="0.4.0"} async-trait = {version="0.1.73"} regex = {version="1.9.4", features=["perf"]} -futures = {version="0.3.28"}dhat = {version="0.3.2", optional = true} +futures = {version="0.3.28"} dhat = {version="0.3.2", optional = true} smallvec = {version="1.11.0", features=["union", "serde"]}