Merge branch 'rolling' into feat-error-box-for-engine-errors
This commit is contained in:
zhou fan
GitHub
commit
575a7f95ba
13 changed files with 344 additions and 349 deletions
|
|
@ -5,55 +5,6 @@ use serde::{Deserialize, Serialize};
|
|||
|
||||
use crate::{config::parser_models::Style, engines::engine_models::EngineError};
|
||||
|
||||
/// A named struct to store, serialize and deserializes the individual search result from all the
|
||||
/// scraped and aggregated search results from the upstream search engines.
|
||||
///
|
||||
/// # Fields
|
||||
///
|
||||
/// * `title` - The title of the search result.
|
||||
/// * `visiting_url` - The url which is accessed when clicked on it (href url in html in simple
|
||||
/// words).
|
||||
/// * `url` - The url to be displayed below the search result title in html.
|
||||
/// * `description` - The description of the search result.
|
||||
/// * `engine` - The names of the upstream engines from which this results were provided.
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SearchResult {
|
||||
pub title: String,
|
||||
pub visiting_url: String,
|
||||
pub url: String,
|
||||
pub description: String,
|
||||
pub engine: Vec<String>,
|
||||
}
|
||||
|
||||
impl SearchResult {
|
||||
/// Constructs a new `SearchResult` with the given arguments needed for the struct.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `title` - The title of the search result.
|
||||
/// * `visiting_url` - The url which is accessed when clicked on it
|
||||
/// (href url in html in simple words).
|
||||
/// * `url` - The url to be displayed below the search result title in html.
|
||||
/// * `description` - The description of the search result.
|
||||
/// * `engine` - The names of the upstream engines from which this results were provided.
|
||||
pub fn new(
|
||||
title: String,
|
||||
visiting_url: String,
|
||||
url: String,
|
||||
description: String,
|
||||
engine: Vec<String>,
|
||||
) -> Self {
|
||||
SearchResult {
|
||||
title,
|
||||
visiting_url,
|
||||
url,
|
||||
description,
|
||||
engine,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A named struct to store the raw scraped search results scraped search results from the
|
||||
/// upstream search engines before aggregating it.It derives the Clone trait which is needed
|
||||
/// to write idiomatic rust using `Iterators`.
|
||||
|
|
@ -61,37 +12,33 @@ impl SearchResult {
|
|||
/// # Fields
|
||||
///
|
||||
/// * `title` - The title of the search result.
|
||||
/// * `visiting_url` - The url which is accessed when clicked on it
|
||||
/// * `url` - The url which is accessed when clicked on it
|
||||
/// (href url in html in simple words).
|
||||
/// * `description` - The description of the search result.
|
||||
/// * `engine` - The names of the upstream engines from which this results were provided.
|
||||
#[derive(Clone)]
|
||||
pub struct RawSearchResult {
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SearchResult {
|
||||
pub title: String,
|
||||
pub visiting_url: String,
|
||||
pub url: String,
|
||||
pub description: String,
|
||||
pub engine: Vec<String>,
|
||||
}
|
||||
|
||||
impl RawSearchResult {
|
||||
impl SearchResult {
|
||||
/// Constructs a new `RawSearchResult` with the given arguments needed for the struct.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `title` - The title of the search result.
|
||||
/// * `visiting_url` - The url which is accessed when clicked on it
|
||||
/// * `url` - The url which is accessed when clicked on it
|
||||
/// (href url in html in simple words).
|
||||
/// * `description` - The description of the search result.
|
||||
/// * `engine` - The names of the upstream engines from which this results were provided.
|
||||
pub fn new(
|
||||
title: String,
|
||||
visiting_url: String,
|
||||
description: String,
|
||||
engine: Vec<String>,
|
||||
) -> Self {
|
||||
RawSearchResult {
|
||||
pub fn new(title: String, url: String, description: String, engine: Vec<String>) -> Self {
|
||||
SearchResult {
|
||||
title,
|
||||
visiting_url,
|
||||
url,
|
||||
description,
|
||||
engine,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,18 +8,14 @@ use rand::Rng;
|
|||
use tokio::task::JoinHandle;
|
||||
|
||||
use super::{
|
||||
aggregation_models::{EngineErrorInfo, RawSearchResult, SearchResult, SearchResults},
|
||||
aggregation_models::{EngineErrorInfo, SearchResult, SearchResults},
|
||||
user_agent::random_user_agent,
|
||||
};
|
||||
|
||||
use crate::engines::{
|
||||
duckduckgo,
|
||||
engine_models::{EngineError, SearchEngine},
|
||||
searx,
|
||||
};
|
||||
use crate::engines::engine_models::{EngineError, EngineHandler};
|
||||
|
||||
/// Aliases for long type annotations
|
||||
type FutureVec = Vec<JoinHandle<Result<HashMap<String, RawSearchResult>, Report<EngineError>>>>;
|
||||
type FutureVec = Vec<JoinHandle<Result<HashMap<String, SearchResult>, Report<EngineError>>>>;
|
||||
|
||||
/// The function aggregates the scraped results from the user-selected upstream search engines.
|
||||
/// These engines can be chosen either from the user interface (UI) or from the configuration file.
|
||||
|
|
@ -64,139 +60,93 @@ pub async fn aggregate(
|
|||
page: u32,
|
||||
random_delay: bool,
|
||||
debug: bool,
|
||||
upstream_search_engines: Vec<String>,
|
||||
upstream_search_engines: Vec<EngineHandler>,
|
||||
request_timeout: u8,
|
||||
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
||||
let user_agent: String = random_user_agent();
|
||||
let mut result_map: HashMap<String, RawSearchResult> = HashMap::new();
|
||||
|
||||
// Add a random delay before making the request.
|
||||
if random_delay || !debug {
|
||||
let mut rng = rand::thread_rng();
|
||||
let delay_secs = rng.gen_range(1..10);
|
||||
std::thread::sleep(Duration::from_secs(delay_secs));
|
||||
tokio::time::sleep(Duration::from_secs(delay_secs)).await;
|
||||
}
|
||||
|
||||
// fetch results from upstream search engines simultaneously/concurrently.
|
||||
let search_engines: Vec<Box<dyn SearchEngine + Send + Sync>> = upstream_search_engines
|
||||
.iter()
|
||||
.map(|engine| match engine.to_lowercase().as_str() {
|
||||
"duckduckgo" => Box::new(duckduckgo::DuckDuckGo) as Box<dyn SearchEngine + Send + Sync>,
|
||||
"searx" => Box::new(searx::Searx) as Box<dyn SearchEngine + Send + Sync>,
|
||||
&_ => panic!("Config Error: Incorrect config file option provided"),
|
||||
})
|
||||
.collect();
|
||||
let mut names: Vec<&str> = vec![];
|
||||
|
||||
let task_capacity: usize = search_engines.len();
|
||||
// create tasks for upstream result fetching
|
||||
let mut tasks: FutureVec = FutureVec::new();
|
||||
|
||||
let tasks: FutureVec = search_engines
|
||||
.into_iter()
|
||||
.map(|search_engine| {
|
||||
let query: String = query.clone();
|
||||
let user_agent: String = user_agent.clone();
|
||||
tokio::spawn(async move {
|
||||
search_engine
|
||||
.results(query, page, user_agent.clone(), request_timeout)
|
||||
.await
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
for engine_handler in upstream_search_engines {
|
||||
let (name, search_engine) = engine_handler.into_name_engine();
|
||||
names.push(name);
|
||||
let query: String = query.clone();
|
||||
let user_agent: String = user_agent.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
search_engine
|
||||
.results(query, page, user_agent.clone(), request_timeout)
|
||||
.await
|
||||
}));
|
||||
}
|
||||
|
||||
let mut outputs = Vec::with_capacity(task_capacity);
|
||||
// get upstream responses
|
||||
let mut responses = Vec::with_capacity(tasks.len());
|
||||
|
||||
for task in tasks {
|
||||
if let Ok(result) = task.await {
|
||||
outputs.push(result)
|
||||
responses.push(result)
|
||||
}
|
||||
}
|
||||
|
||||
// aggregate search results, removing duplicates and handling errors the upstream engines returned
|
||||
let mut result_map: HashMap<String, SearchResult> = HashMap::new();
|
||||
let mut engine_errors_info: Vec<EngineErrorInfo> = Vec::new();
|
||||
|
||||
// The code block `outputs.iter()` determines whether it is the first time the code is being run.
|
||||
// It does this by checking the initial flag. If it is the first time, the code selects the first
|
||||
// engine from which results are fetched and adds or extends them into the `result_map`. If the
|
||||
// initially selected engine fails, the code automatically selects another engine to map or extend
|
||||
// into the `result_map`. On the other hand, if an engine selected for the first time successfully
|
||||
// fetches results and maps them into the `result_map`, the initial flag is set to false. Subsequently,
|
||||
// the code iterates through the remaining engines one by one. It compares the fetched results from each
|
||||
// engine with the results already present in the `result_map` to identify any duplicates. If duplicate
|
||||
// results are found, the code groups them together with the name of the engine from which they were
|
||||
// fetched, and automatically removes the duplicate results from the newly fetched data.
|
||||
//
|
||||
// Additionally, the code handles errors returned by the engines. It keeps track of which engines
|
||||
// encountered errors and stores this information in a vector of structures called `EngineErrorInfo`.
|
||||
// Each structure in this vector contains the name of the engine and the type of error it returned.
|
||||
// These structures will later be added to the final `SearchResults` structure. The `SearchResults`
|
||||
// structure is used to display an error box in the UI containing the relevant information from
|
||||
// the `EngineErrorInfo` structure.
|
||||
//
|
||||
// In summary, this code block manages the selection of engines, handling of duplicate results, and tracking
|
||||
// of errors in order to populate the `result_map` and provide informative feedback to the user through the
|
||||
// `SearchResults` structure.
|
||||
let mut initial: bool = true;
|
||||
let mut counter: usize = 0;
|
||||
outputs.iter().for_each(|results| {
|
||||
if initial {
|
||||
match results {
|
||||
Ok(result) => {
|
||||
result_map.extend(result.clone());
|
||||
counter += 1;
|
||||
initial = false
|
||||
let mut handle_error = |error: Report<EngineError>, engine_name: String| {
|
||||
log::error!("Engine Error: {:?}", error);
|
||||
engine_errors_info.push(EngineErrorInfo::new(
|
||||
error.downcast_ref::<EngineError>().unwrap(),
|
||||
engine_name.to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
for _ in 0..responses.len() {
|
||||
let response = responses.pop().unwrap();
|
||||
let engine = names.pop().unwrap().to_string();
|
||||
|
||||
if result_map.is_empty() {
|
||||
match response {
|
||||
Ok(results) => {
|
||||
result_map = results.clone();
|
||||
}
|
||||
Err(error_type) => {
|
||||
log::error!("Engine Error: {:?}", error_type);
|
||||
engine_errors_info.push(EngineErrorInfo::new(
|
||||
error_type.downcast_ref::<EngineError>().unwrap(),
|
||||
upstream_search_engines[counter].clone(),
|
||||
));
|
||||
counter += 1
|
||||
Err(error) => {
|
||||
handle_error(error, engine);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
match results {
|
||||
Ok(result) => {
|
||||
result.clone().into_iter().for_each(|(key, value)| {
|
||||
result_map
|
||||
.entry(key)
|
||||
.and_modify(|result| {
|
||||
result.add_engines(value.clone().engine());
|
||||
})
|
||||
.or_insert_with(|| -> RawSearchResult {
|
||||
RawSearchResult::new(
|
||||
value.title.clone(),
|
||||
value.visiting_url.clone(),
|
||||
value.description.clone(),
|
||||
value.engine.clone(),
|
||||
)
|
||||
});
|
||||
});
|
||||
counter += 1
|
||||
}
|
||||
Err(error_type) => {
|
||||
log::error!("Engine Error: {:?}", error_type);
|
||||
engine_errors_info.push(EngineErrorInfo::new(
|
||||
error_type.downcast_ref::<EngineError>().unwrap(),
|
||||
upstream_search_engines[counter].clone(),
|
||||
));
|
||||
counter += 1
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
match response {
|
||||
Ok(result) => {
|
||||
result.into_iter().for_each(|(key, value)| {
|
||||
result_map
|
||||
.entry(key)
|
||||
.and_modify(|result| {
|
||||
result.add_engines(engine.clone());
|
||||
})
|
||||
.or_insert_with(|| -> SearchResult { value });
|
||||
});
|
||||
}
|
||||
Err(error) => {
|
||||
handle_error(error, engine);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = result_map.into_values().collect();
|
||||
|
||||
Ok(SearchResults::new(
|
||||
result_map
|
||||
.into_iter()
|
||||
.map(|(key, value)| {
|
||||
SearchResult::new(
|
||||
value.title,
|
||||
value.visiting_url,
|
||||
key,
|
||||
value.description,
|
||||
value.engine,
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
results,
|
||||
query.to_string(),
|
||||
engine_errors_info,
|
||||
))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue