morrigan/crabbysearch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

optimise code for large scale server use and closes #7

Browse source
This commit is contained in:
neon_arch 2023-05-07 21:18:19 +03:00
parent 3357fa2647
commit c5c1684cc1
3 changed files with 22 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/engines/duckduckgo.rs
View file

@ -2,9 +2,8 @@
//! by querying the upstream duckduckgo search engine with user provided query and with a page
//! number if provided.
use std::{collections::HashMap, time::Duration};
use std::collections::HashMap;
use rand::Rng;
use reqwest::header::{HeaderMap, CONTENT_TYPE, REFERER, USER_AGENT};
use scraper::{Html, Selector};
@ -47,11 +46,6 @@ pub async fn results(
}
};
// Add a random delay before making the request.
let mut rng = rand::thread_rng();
let delay_secs = rng.gen_range(1, 10);
std::thread::sleep(Duration::from_secs(delay_secs));
// initializing HeaderMap and adding appropriate headers.
let mut header_map = HeaderMap::new();
header_map.insert(USER_AGENT, user_agent.parse()?);

8
src/engines/searx.rs
View file

@ -2,10 +2,9 @@
//! by querying the upstream searx search engine instance with user provided query and with a page
//! number if provided.
use rand::Rng;
use reqwest::header::{HeaderMap, CONTENT_TYPE, REFERER, USER_AGENT};
use scraper::{Html, Selector};
use std::{collections::HashMap, time::Duration};
use std::collections::HashMap;
use crate::search_results_handler::aggregation_models::RawSearchResult;
@ -34,11 +33,6 @@ pub async fn results(
// so that upstream server recieves valid page number.
let url: String = format!("https://searx.work/search?q={query}&pageno={page}");
// Add random delay before making the request.
let mut rng = rand::thread_rng();
let delay_secs = rng.gen_range(1, 10);
std::thread::sleep(Duration::from_secs(delay_secs));
// initializing headers and adding appropriate headers.
let mut header_map = HeaderMap::new();
header_map.insert(USER_AGENT, user_agent.parse()?);