Merge branch 'rolling' into patch-searx-results-on-first-page

src/engines/duckduckgo.rs

@@ -29,6 +29,7 @@ impl SearchEngine for DuckDuckGo {
     /// * `query` - Takes the user provided query to query to the upstream search engine with.
     /// * `page` - Takes an u32 as an argument.
     /// * `user_agent` - Takes a random user agent string as an argument.
+    /// * `request_timeout` - Takes a time (secs) as a value which controls the server request timeout.
     ///
     /// # Errors
     ///
@@ -41,6 +42,7 @@ impl SearchEngine for DuckDuckGo {
         query: String,
         page: u32,
         user_agent: String,
+        request_timeout: u8,
     ) -> Result<HashMap<String, RawSearchResult>, EngineError> {
         // Page number can be missing or empty string and so appropriate handling is required
         // so that upstream server recieves valid page number.
@@ -90,7 +92,7 @@ impl SearchEngine for DuckDuckGo {
         );
 
         let document: Html = Html::parse_document(
-            &DuckDuckGo::fetch_html_from_upstream(self, url, header_map).await?,
+            &DuckDuckGo::fetch_html_from_upstream(self, url, header_map, request_timeout).await?,
         );
 
         let no_result: Selector = Selector::parse(".no-results")

src/engines/engine_models.rs

@@ -50,11 +50,12 @@ pub trait SearchEngine {
         &self,
         url: String,
         header_map: reqwest::header::HeaderMap,
+        request_timeout: u8,
     ) -> Result<String, EngineError> {
         // fetch the html from upstream search engine
         Ok(reqwest::Client::new()
             .get(url)
-            .timeout(Duration::from_secs(30)) // Add timeout to request to avoid DDOSing the server
+            .timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
             .headers(header_map) // add spoofed headers to emulate human behaviour
             .send()
             .await
@@ -71,5 +72,6 @@ pub trait SearchEngine {
         query: String,
         page: u32,
         user_agent: String,
+        request_timeout: u8,
     ) -> Result<HashMap<String, RawSearchResult>, EngineError>;
 }

src/engines/searx.rs

@@ -27,6 +27,7 @@ impl SearchEngine for Searx {
     /// * `query` - Takes the user provided query to query to the upstream search engine with.
     /// * `page` - Takes an u32 as an argument.
     /// * `user_agent` - Takes a random user agent string as an argument.
+    /// * `request_timeout` - Takes a time (secs) as a value which controls the server request timeout.
     ///
     /// # Errors
     ///
@@ -40,6 +41,7 @@ impl SearchEngine for Searx {
         query: String,
         page: u32,
         user_agent: String,
+        request_timeout: u8,
     ) -> Result<HashMap<String, RawSearchResult>, EngineError> {
         // Page number can be missing or empty string and so appropriate handling is required
         // so that upstream server recieves valid page number.
@@ -73,8 +75,9 @@ impl SearchEngine for Searx {
         );
         header_map.insert(COOKIE, "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".parse().into_report().change_context(EngineError::UnexpectedError)?);
 
-        let document: Html =
-            Html::parse_document(&Searx::fetch_html_from_upstream(self, url, header_map).await?);
+        let document: Html = Html::parse_document(
+            &Searx::fetch_html_from_upstream(self, url, header_map, request_timeout).await?,
+        );
 
         let no_result: Selector = Selector::parse("#urls>.dialog-error>p")
             .map_err(|_| Report::new(EngineError::UnexpectedError))