Improve Aggregation function & config parser
Refactor aggregation function Rename visiting_url to url, as they are always the same (see upstream engine scalping). Refactor parsing function to be more readable.
This commit is contained in:
parent
af3b1cb308
commit
15dfda6ea9
8 changed files with 103 additions and 136 deletions
|
@ -147,7 +147,7 @@ impl SearchEngine for DuckDuckGo {
|
|||
vec!["duckduckgo".to_string()],
|
||||
)
|
||||
})
|
||||
.map(|search_result| (search_result.visiting_url.clone(), search_result))
|
||||
.map(|search_result| (search_result.url.clone(), search_result))
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
|
|
@ -43,7 +43,7 @@ impl fmt::Display for EngineError {
|
|||
|
||||
impl error_stack::Context for EngineError {}
|
||||
|
||||
/// A trait to define common behaviour for all search engines.
|
||||
/// A trait to define common behavior for all search engines.
|
||||
#[async_trait::async_trait]
|
||||
pub trait SearchEngine {
|
||||
async fn fetch_html_from_upstream(
|
||||
|
@ -56,7 +56,7 @@ pub trait SearchEngine {
|
|||
Ok(reqwest::Client::new()
|
||||
.get(url)
|
||||
.timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
|
||||
.headers(header_map) // add spoofed headers to emulate human behaviour
|
||||
.headers(header_map) // add spoofed headers to emulate human behavior
|
||||
.send()
|
||||
.await
|
||||
.into_report()
|
||||
|
|
|
@ -137,7 +137,7 @@ impl SearchEngine for Searx {
|
|||
vec!["searx".to_string()],
|
||||
)
|
||||
})
|
||||
.map(|search_result| (search_result.visiting_url.clone(), search_result))
|
||||
.map(|search_result| (search_result.url.clone(), search_result))
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue