diff --git a/src/models/engine_models.rs b/src/models/engine_models.rs index 98367e8..f8e966e 100644 --- a/src/models/engine_models.rs +++ b/src/models/engine_models.rs @@ -3,7 +3,8 @@ use super::aggregation_models::SearchResult; use error_stack::{Report, Result, ResultExt}; -use std::{collections::HashMap, fmt, time::Duration}; +use reqwest::Client; +use std::{collections::HashMap, fmt}; /// A custom error type used for handle engine associated errors. #[derive(Debug)] @@ -71,12 +72,11 @@ pub trait SearchEngine: Sync + Send { &self, url: &str, header_map: reqwest::header::HeaderMap, - request_timeout: u8, + client: &Client, ) -> Result { // fetch the html from upstream search engine - Ok(reqwest::Client::new() + Ok(client .get(url) - .timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server .headers(header_map) // add spoofed headers to emulate human behavior .send() .await @@ -109,7 +109,7 @@ pub trait SearchEngine: Sync + Send { query: &str, page: u32, user_agent: &str, - request_timeout: u8, + client: &Client, safe_search: u8, ) -> Result, EngineError>; } diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index b57768f..3285786 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -9,6 +9,7 @@ use crate::models::{ }; use error_stack::Report; use regex::Regex; +use reqwest::{Client, ClientBuilder}; use std::time::{SystemTime, UNIX_EPOCH}; use std::{ collections::HashMap, @@ -18,6 +19,9 @@ use std::{ use std::{fs::File, io::BufRead}; use tokio::task::JoinHandle; +/// A constant for holding the prebuilt Client globally in the app. +static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); + /// Aliases for long type annotations type FutureVec = Vec, Report>>>; @@ -68,6 +72,16 @@ pub async fn aggregate( request_timeout: u8, safe_search: u8, ) -> Result> { + let client = CLIENT.get_or_init(|| { + ClientBuilder::new() + .timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server + .https_only(true) + .gzip(true) + .brotli(true) + .build() + .unwrap() + }); + let user_agent: &str = random_user_agent(); // Add a random delay before making the request. @@ -88,7 +102,7 @@ pub async fn aggregate( let query: String = query.to_owned(); tasks.push(tokio::spawn(async move { search_engine - .results(&query, page, user_agent, request_timeout, safe_search) + .results(&query, page, user_agent, client, safe_search) .await })); }