Improve Aggregation function & config parser

Refactor aggregation function
Rename visiting_url to url, as they are always the same (see upstream engine scalping).
Refactor parsing function to be more readable.
This commit is contained in:
Milim 2023-08-17 22:48:20 +02:00
parent af3b1cb308
commit 15dfda6ea9
8 changed files with 103 additions and 136 deletions

View file

@ -147,7 +147,7 @@ impl SearchEngine for DuckDuckGo {
vec!["duckduckgo".to_string()],
)
})
.map(|search_result| (search_result.visiting_url.clone(), search_result))
.map(|search_result| (search_result.url.clone(), search_result))
.collect())
}
}

View file

@ -43,7 +43,7 @@ impl fmt::Display for EngineError {
impl error_stack::Context for EngineError {}
/// A trait to define common behaviour for all search engines.
/// A trait to define common behavior for all search engines.
#[async_trait::async_trait]
pub trait SearchEngine {
async fn fetch_html_from_upstream(
@ -56,7 +56,7 @@ pub trait SearchEngine {
Ok(reqwest::Client::new()
.get(url)
.timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
.headers(header_map) // add spoofed headers to emulate human behaviour
.headers(header_map) // add spoofed headers to emulate human behavior
.send()
.await
.into_report()

View file

@ -137,7 +137,7 @@ impl SearchEngine for Searx {
vec!["searx".to_string()],
)
})
.map(|search_result| (search_result.visiting_url.clone(), search_result))
.map(|search_result| (search_result.url.clone(), search_result))
.collect())
}
}